aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--MAINTAINERS1
-rw-r--r--drivers/infiniband/core/uverbs_cmd.c2
-rw-r--r--drivers/infiniband/core/uverbs_main.c5
-rw-r--r--drivers/infiniband/hw/cxgb4/cm.c32
-rw-r--r--drivers/infiniband/hw/cxgb4/device.c2
-rw-r--r--drivers/infiniband/hw/mlx5/main.c8
-rw-r--r--drivers/infiniband/hw/mlx5/mem.c18
-rw-r--r--drivers/infiniband/hw/mlx5/mr.c6
-rw-r--r--drivers/infiniband/hw/mlx5/qp.c149
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_hw.c25
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_main.c12
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_sli.h238
-rw-r--r--drivers/infiniband/hw/ocrdma/ocrdma_verbs.c10
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.c313
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.h408
-rw-r--r--drivers/infiniband/ulp/iser/iser_initiator.c198
-rw-r--r--drivers/infiniband/ulp/iser/iser_memory.c99
-rw-r--r--drivers/infiniband/ulp/iser/iser_verbs.c667
-rw-r--r--drivers/infiniband/ulp/isert/ib_isert.c65
-rw-r--r--include/linux/mlx5/qp.h35
-rw-r--r--include/rdma/ib_verbs.h32
21 files changed, 1372 insertions, 953 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index 33c0d433d554..43898b1a8a2d 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -5112,6 +5112,7 @@ F: include/scsi/*iscsi*
5112 5112
5113ISCSI EXTENSIONS FOR RDMA (ISER) INITIATOR 5113ISCSI EXTENSIONS FOR RDMA (ISER) INITIATOR
5114M: Or Gerlitz <ogerlitz@mellanox.com> 5114M: Or Gerlitz <ogerlitz@mellanox.com>
5115M: Sagi Grimberg <sagig@mellanox.com>
5115M: Roi Dayan <roid@mellanox.com> 5116M: Roi Dayan <roid@mellanox.com>
5116L: linux-rdma@vger.kernel.org 5117L: linux-rdma@vger.kernel.org
5117S: Supported 5118S: Supported
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 0600c50e6215..5ba2a86aab6a 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -2518,6 +2518,8 @@ ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file,
2518 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 2518 attr.grh.sgid_index = cmd.attr.grh.sgid_index;
2519 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 2519 attr.grh.hop_limit = cmd.attr.grh.hop_limit;
2520 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 2520 attr.grh.traffic_class = cmd.attr.grh.traffic_class;
2521 attr.vlan_id = 0;
2522 memset(&attr.dmac, 0, sizeof(attr.dmac));
2521 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 2523 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16);
2522 2524
2523 ah = ib_create_ah(pd, &attr); 2525 ah = ib_create_ah(pd, &attr);
diff --git a/drivers/infiniband/core/uverbs_main.c b/drivers/infiniband/core/uverbs_main.c
index c73b22a257fe..71ab83fde472 100644
--- a/drivers/infiniband/core/uverbs_main.c
+++ b/drivers/infiniband/core/uverbs_main.c
@@ -477,6 +477,7 @@ static void ib_uverbs_async_handler(struct ib_uverbs_file *file,
477 477
478 entry->desc.async.element = element; 478 entry->desc.async.element = element;
479 entry->desc.async.event_type = event; 479 entry->desc.async.event_type = event;
480 entry->desc.async.reserved = 0;
480 entry->counter = counter; 481 entry->counter = counter;
481 482
482 list_add_tail(&entry->list, &file->async_file->event_list); 483 list_add_tail(&entry->list, &file->async_file->event_list);
@@ -502,6 +503,10 @@ void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
502{ 503{
503 struct ib_uevent_object *uobj; 504 struct ib_uevent_object *uobj;
504 505
506 /* for XRC target qp's, check that qp is live */
507 if (!event->element.qp->uobject || !event->element.qp->uobject->live)
508 return;
509
505 uobj = container_of(event->element.qp->uobject, 510 uobj = container_of(event->element.qp->uobject,
506 struct ib_uevent_object, uobject); 511 struct ib_uevent_object, uobject);
507 512
diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c
index c2fb71c182a8..fb61f6685809 100644
--- a/drivers/infiniband/hw/cxgb4/cm.c
+++ b/drivers/infiniband/hw/cxgb4/cm.c
@@ -236,10 +236,12 @@ static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb)
236static void set_emss(struct c4iw_ep *ep, u16 opt) 236static void set_emss(struct c4iw_ep *ep, u16 opt)
237{ 237{
238 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 238 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] -
239 sizeof(struct iphdr) - sizeof(struct tcphdr); 239 ((AF_INET == ep->com.remote_addr.ss_family) ?
240 sizeof(struct iphdr) : sizeof(struct ipv6hdr)) -
241 sizeof(struct tcphdr);
240 ep->mss = ep->emss; 242 ep->mss = ep->emss;
241 if (GET_TCPOPT_TSTAMP(opt)) 243 if (GET_TCPOPT_TSTAMP(opt))
242 ep->emss -= 12; 244 ep->emss -= round_up(TCPOLEN_TIMESTAMP, 4);
243 if (ep->emss < 128) 245 if (ep->emss < 128)
244 ep->emss = 128; 246 ep->emss = 128;
245 if (ep->emss & 7) 247 if (ep->emss & 7)
@@ -415,6 +417,7 @@ static struct dst_entry *find_route(struct c4iw_dev *dev, __be32 local_ip,
415 return NULL; 417 return NULL;
416 if (!our_interface(dev, n->dev) && 418 if (!our_interface(dev, n->dev) &&
417 !(n->dev->flags & IFF_LOOPBACK)) { 419 !(n->dev->flags & IFF_LOOPBACK)) {
420 neigh_release(n);
418 dst_release(&rt->dst); 421 dst_release(&rt->dst);
419 return NULL; 422 return NULL;
420 } 423 }
@@ -581,11 +584,14 @@ static void c4iw_record_pm_msg(struct c4iw_ep *ep,
581} 584}
582 585
583static void best_mtu(const unsigned short *mtus, unsigned short mtu, 586static void best_mtu(const unsigned short *mtus, unsigned short mtu,
584 unsigned int *idx, int use_ts) 587 unsigned int *idx, int use_ts, int ipv6)
585{ 588{
586 unsigned short hdr_size = sizeof(struct iphdr) + 589 unsigned short hdr_size = (ipv6 ?
590 sizeof(struct ipv6hdr) :
591 sizeof(struct iphdr)) +
587 sizeof(struct tcphdr) + 592 sizeof(struct tcphdr) +
588 (use_ts ? 12 : 0); 593 (use_ts ?
594 round_up(TCPOLEN_TIMESTAMP, 4) : 0);
589 unsigned short data_size = mtu - hdr_size; 595 unsigned short data_size = mtu - hdr_size;
590 596
591 cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx); 597 cxgb4_best_aligned_mtu(mtus, hdr_size, data_size, 8, idx);
@@ -634,7 +640,8 @@ static int send_connect(struct c4iw_ep *ep)
634 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 640 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx);
635 641
636 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx, 642 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
637 enable_tcp_timestamps); 643 enable_tcp_timestamps,
644 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
638 wscale = compute_wscale(rcv_win); 645 wscale = compute_wscale(rcv_win);
639 646
640 /* 647 /*
@@ -668,6 +675,7 @@ static int send_connect(struct c4iw_ep *ep)
668 if (is_t5(ep->com.dev->rdev.lldi.adapter_type)) { 675 if (is_t5(ep->com.dev->rdev.lldi.adapter_type)) {
669 opt2 |= T5_OPT_2_VALID; 676 opt2 |= T5_OPT_2_VALID;
670 opt2 |= V_CONG_CNTRL(CONG_ALG_TAHOE); 677 opt2 |= V_CONG_CNTRL(CONG_ALG_TAHOE);
678 opt2 |= CONG_CNTRL_VALID; /* OPT_2_ISS for T5 */
671 } 679 }
672 t4_set_arp_err_handler(skb, ep, act_open_req_arp_failure); 680 t4_set_arp_err_handler(skb, ep, act_open_req_arp_failure);
673 681
@@ -713,8 +721,6 @@ static int send_connect(struct c4iw_ep *ep)
713 } else { 721 } else {
714 u32 isn = (prandom_u32() & ~7UL) - 1; 722 u32 isn = (prandom_u32() & ~7UL) - 1;
715 723
716 opt2 |= T5_OPT_2_VALID;
717 opt2 |= CONG_CNTRL_VALID; /* OPT_2_ISS for T5 */
718 if (peer2peer) 724 if (peer2peer)
719 isn += 4; 725 isn += 4;
720 726
@@ -756,10 +762,10 @@ static int send_connect(struct c4iw_ep *ep)
756 t5_req6->peer_ip_lo = *((__be64 *) 762 t5_req6->peer_ip_lo = *((__be64 *)
757 (ra6->sin6_addr.s6_addr + 8)); 763 (ra6->sin6_addr.s6_addr + 8));
758 t5_req6->opt0 = cpu_to_be64(opt0); 764 t5_req6->opt0 = cpu_to_be64(opt0);
759 t5_req6->params = (__force __be64)cpu_to_be32( 765 t5_req6->params = cpu_to_be64(V_FILTER_TUPLE(
760 cxgb4_select_ntuple( 766 cxgb4_select_ntuple(
761 ep->com.dev->rdev.lldi.ports[0], 767 ep->com.dev->rdev.lldi.ports[0],
762 ep->l2t)); 768 ep->l2t)));
763 t5_req6->rsvd = cpu_to_be32(isn); 769 t5_req6->rsvd = cpu_to_be32(isn);
764 PDBG("%s snd_isn %u\n", __func__, 770 PDBG("%s snd_isn %u\n", __func__,
765 be32_to_cpu(t5_req6->rsvd)); 771 be32_to_cpu(t5_req6->rsvd));
@@ -1763,7 +1769,8 @@ static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid)
1763 req->tcb.tx_max = (__force __be32) jiffies; 1769 req->tcb.tx_max = (__force __be32) jiffies;
1764 req->tcb.rcv_adv = htons(1); 1770 req->tcb.rcv_adv = htons(1);
1765 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx, 1771 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
1766 enable_tcp_timestamps); 1772 enable_tcp_timestamps,
1773 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
1767 wscale = compute_wscale(rcv_win); 1774 wscale = compute_wscale(rcv_win);
1768 1775
1769 /* 1776 /*
@@ -2162,7 +2169,8 @@ static void accept_cr(struct c4iw_ep *ep, struct sk_buff *skb,
2162 ep->hwtid)); 2169 ep->hwtid));
2163 2170
2164 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx, 2171 best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx,
2165 enable_tcp_timestamps && req->tcpopt.tstamp); 2172 enable_tcp_timestamps && req->tcpopt.tstamp,
2173 (AF_INET == ep->com.remote_addr.ss_family) ? 0 : 1);
2166 wscale = compute_wscale(rcv_win); 2174 wscale = compute_wscale(rcv_win);
2167 2175
2168 /* 2176 /*
diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c
index f25df5276c22..72f1f052e88c 100644
--- a/drivers/infiniband/hw/cxgb4/device.c
+++ b/drivers/infiniband/hw/cxgb4/device.c
@@ -60,7 +60,7 @@ int c4iw_wr_log = 0;
60module_param(c4iw_wr_log, int, 0444); 60module_param(c4iw_wr_log, int, 0444);
61MODULE_PARM_DESC(c4iw_wr_log, "Enables logging of work request timing data."); 61MODULE_PARM_DESC(c4iw_wr_log, "Enables logging of work request timing data.");
62 62
63int c4iw_wr_log_size_order = 12; 63static int c4iw_wr_log_size_order = 12;
64module_param(c4iw_wr_log_size_order, int, 0444); 64module_param(c4iw_wr_log_size_order, int, 0444);
65MODULE_PARM_DESC(c4iw_wr_log_size_order, 65MODULE_PARM_DESC(c4iw_wr_log_size_order,
66 "Number of entries (log2) in the work request timing log."); 66 "Number of entries (log2) in the work request timing log.");
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index f3114d1132fb..1ba6c42e4df8 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -657,13 +657,13 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
657 return -EINVAL; 657 return -EINVAL;
658 658
659 idx = get_index(vma->vm_pgoff); 659 idx = get_index(vma->vm_pgoff);
660 if (idx >= uuari->num_uars)
661 return -EINVAL;
662
660 pfn = uar_index2pfn(dev, uuari->uars[idx].index); 663 pfn = uar_index2pfn(dev, uuari->uars[idx].index);
661 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx, 664 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx,
662 (unsigned long long)pfn); 665 (unsigned long long)pfn);
663 666
664 if (idx >= uuari->num_uars)
665 return -EINVAL;
666
667 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); 667 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
668 if (io_remap_pfn_range(vma, vma->vm_start, pfn, 668 if (io_remap_pfn_range(vma, vma->vm_start, pfn,
669 PAGE_SIZE, vma->vm_page_prot)) 669 PAGE_SIZE, vma->vm_page_prot))
@@ -1425,8 +1425,8 @@ err_dealloc:
1425static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context) 1425static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
1426{ 1426{
1427 struct mlx5_ib_dev *dev = context; 1427 struct mlx5_ib_dev *dev = context;
1428 destroy_umrc_res(dev);
1429 ib_unregister_device(&dev->ib_dev); 1428 ib_unregister_device(&dev->ib_dev);
1429 destroy_umrc_res(dev);
1430 destroy_dev_resources(&dev->devr); 1430 destroy_dev_resources(&dev->devr);
1431 free_comp_eqs(dev); 1431 free_comp_eqs(dev);
1432 ib_dealloc_device(&dev->ib_dev); 1432 ib_dealloc_device(&dev->ib_dev);
diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c
index a3e81444c825..dae07eae9507 100644
--- a/drivers/infiniband/hw/mlx5/mem.c
+++ b/drivers/infiniband/hw/mlx5/mem.c
@@ -55,16 +55,17 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
55 u64 pfn; 55 u64 pfn;
56 struct scatterlist *sg; 56 struct scatterlist *sg;
57 int entry; 57 int entry;
58 unsigned long page_shift = ilog2(umem->page_size);
58 59
59 addr = addr >> PAGE_SHIFT; 60 addr = addr >> page_shift;
60 tmp = (unsigned long)addr; 61 tmp = (unsigned long)addr;
61 m = find_first_bit(&tmp, sizeof(tmp)); 62 m = find_first_bit(&tmp, sizeof(tmp));
62 skip = 1 << m; 63 skip = 1 << m;
63 mask = skip - 1; 64 mask = skip - 1;
64 i = 0; 65 i = 0;
65 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 66 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
66 len = sg_dma_len(sg) >> PAGE_SHIFT; 67 len = sg_dma_len(sg) >> page_shift;
67 pfn = sg_dma_address(sg) >> PAGE_SHIFT; 68 pfn = sg_dma_address(sg) >> page_shift;
68 for (k = 0; k < len; k++) { 69 for (k = 0; k < len; k++) {
69 if (!(i & mask)) { 70 if (!(i & mask)) {
70 tmp = (unsigned long)pfn; 71 tmp = (unsigned long)pfn;
@@ -103,14 +104,15 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
103 104
104 *ncont = 0; 105 *ncont = 0;
105 } 106 }
106 *shift = PAGE_SHIFT + m; 107 *shift = page_shift + m;
107 *count = i; 108 *count = i;
108} 109}
109 110
110void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, 111void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
111 int page_shift, __be64 *pas, int umr) 112 int page_shift, __be64 *pas, int umr)
112{ 113{
113 int shift = page_shift - PAGE_SHIFT; 114 unsigned long umem_page_shift = ilog2(umem->page_size);
115 int shift = page_shift - umem_page_shift;
114 int mask = (1 << shift) - 1; 116 int mask = (1 << shift) - 1;
115 int i, k; 117 int i, k;
116 u64 cur = 0; 118 u64 cur = 0;
@@ -121,11 +123,11 @@ void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
121 123
122 i = 0; 124 i = 0;
123 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { 125 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
124 len = sg_dma_len(sg) >> PAGE_SHIFT; 126 len = sg_dma_len(sg) >> umem_page_shift;
125 base = sg_dma_address(sg); 127 base = sg_dma_address(sg);
126 for (k = 0; k < len; k++) { 128 for (k = 0; k < len; k++) {
127 if (!(i & mask)) { 129 if (!(i & mask)) {
128 cur = base + (k << PAGE_SHIFT); 130 cur = base + (k << umem_page_shift);
129 if (umr) 131 if (umr)
130 cur |= 3; 132 cur |= 3;
131 133
@@ -134,7 +136,7 @@ void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
134 i >> shift, be64_to_cpu(pas[i >> shift])); 136 i >> shift, be64_to_cpu(pas[i >> shift]));
135 } else 137 } else
136 mlx5_ib_dbg(dev, "=====> 0x%llx\n", 138 mlx5_ib_dbg(dev, "=====> 0x%llx\n",
137 base + (k << PAGE_SHIFT)); 139 base + (k << umem_page_shift));
138 i++; 140 i++;
139 } 141 }
140 } 142 }
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 80b3c63eab5d..8ee7cb46e059 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -881,12 +881,12 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
881 int order; 881 int order;
882 int err; 882 int err;
883 883
884 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx\n", 884 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n",
885 start, virt_addr, length); 885 start, virt_addr, length, access_flags);
886 umem = ib_umem_get(pd->uobject->context, start, length, access_flags, 886 umem = ib_umem_get(pd->uobject->context, start, length, access_flags,
887 0); 887 0);
888 if (IS_ERR(umem)) { 888 if (IS_ERR(umem)) {
889 mlx5_ib_dbg(dev, "umem get failed\n"); 889 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(umem));
890 return (void *)umem; 890 return (void *)umem;
891 } 891 }
892 892
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index dbfe498870c1..e261a53f9a02 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -1317,6 +1317,11 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, const struct ib_ah_attr *ah,
1317 path->rlid = cpu_to_be16(ah->dlid); 1317 path->rlid = cpu_to_be16(ah->dlid);
1318 1318
1319 if (ah->ah_flags & IB_AH_GRH) { 1319 if (ah->ah_flags & IB_AH_GRH) {
1320 if (ah->grh.sgid_index >= gen->port[port - 1].gid_table_len) {
1321 pr_err(KERN_ERR "sgid_index (%u) too large. max is %d\n",
1322 ah->grh.sgid_index, gen->port[port - 1].gid_table_len);
1323 return -EINVAL;
1324 }
1320 path->grh_mlid |= 1 << 7; 1325 path->grh_mlid |= 1 << 7;
1321 path->mgid_index = ah->grh.sgid_index; 1326 path->mgid_index = ah->grh.sgid_index;
1322 path->hop_limit = ah->grh.hop_limit; 1327 path->hop_limit = ah->grh.hop_limit;
@@ -1332,22 +1337,6 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, const struct ib_ah_attr *ah,
1332 path->static_rate = err; 1337 path->static_rate = err;
1333 path->port = port; 1338 path->port = port;
1334 1339
1335 if (ah->ah_flags & IB_AH_GRH) {
1336 if (ah->grh.sgid_index >= gen->port[port - 1].gid_table_len) {
1337 pr_err(KERN_ERR "sgid_index (%u) too large. max is %d\n",
1338 ah->grh.sgid_index, gen->port[port - 1].gid_table_len);
1339 return -EINVAL;
1340 }
1341
1342 path->grh_mlid |= 1 << 7;
1343 path->mgid_index = ah->grh.sgid_index;
1344 path->hop_limit = ah->grh.hop_limit;
1345 path->tclass_flowlabel =
1346 cpu_to_be32((ah->grh.traffic_class << 20) |
1347 (ah->grh.flow_label));
1348 memcpy(path->rgid, ah->grh.dgid.raw, 16);
1349 }
1350
1351 if (attr_mask & IB_QP_TIMEOUT) 1340 if (attr_mask & IB_QP_TIMEOUT)
1352 path->ackto_lt = attr->timeout << 3; 1341 path->ackto_lt = attr->timeout << 3;
1353 1342
@@ -2039,56 +2028,31 @@ static u8 bs_selector(int block_size)
2039 } 2028 }
2040} 2029}
2041 2030
2042static int format_selector(struct ib_sig_attrs *attr, 2031static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain,
2043 struct ib_sig_domain *domain, 2032 struct mlx5_bsf_inl *inl)
2044 int *selector)
2045{ 2033{
2034 /* Valid inline section and allow BSF refresh */
2035 inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID |
2036 MLX5_BSF_REFRESH_DIF);
2037 inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag);
2038 inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag);
2039 /* repeating block */
2040 inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK;
2041 inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ?
2042 MLX5_DIF_CRC : MLX5_DIF_IPCS;
2046 2043
2047#define FORMAT_DIF_NONE 0 2044 if (domain->sig.dif.ref_remap)
2048#define FORMAT_DIF_CRC_INC 8 2045 inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG;
2049#define FORMAT_DIF_CRC_NO_INC 12
2050#define FORMAT_DIF_CSUM_INC 13
2051#define FORMAT_DIF_CSUM_NO_INC 14
2052 2046
2053 switch (domain->sig.dif.type) { 2047 if (domain->sig.dif.app_escape) {
2054 case IB_T10DIF_NONE: 2048 if (domain->sig.dif.ref_escape)
2055 /* No DIF */ 2049 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE;
2056 *selector = FORMAT_DIF_NONE; 2050 else
2057 break; 2051 inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE;
2058 case IB_T10DIF_TYPE1: /* Fall through */
2059 case IB_T10DIF_TYPE2:
2060 switch (domain->sig.dif.bg_type) {
2061 case IB_T10DIF_CRC:
2062 *selector = FORMAT_DIF_CRC_INC;
2063 break;
2064 case IB_T10DIF_CSUM:
2065 *selector = FORMAT_DIF_CSUM_INC;
2066 break;
2067 default:
2068 return 1;
2069 }
2070 break;
2071 case IB_T10DIF_TYPE3:
2072 switch (domain->sig.dif.bg_type) {
2073 case IB_T10DIF_CRC:
2074 *selector = domain->sig.dif.type3_inc_reftag ?
2075 FORMAT_DIF_CRC_INC :
2076 FORMAT_DIF_CRC_NO_INC;
2077 break;
2078 case IB_T10DIF_CSUM:
2079 *selector = domain->sig.dif.type3_inc_reftag ?
2080 FORMAT_DIF_CSUM_INC :
2081 FORMAT_DIF_CSUM_NO_INC;
2082 break;
2083 default:
2084 return 1;
2085 }
2086 break;
2087 default:
2088 return 1;
2089 } 2052 }
2090 2053
2091 return 0; 2054 inl->dif_app_bitmask_check =
2055 cpu_to_be16(domain->sig.dif.apptag_check_mask);
2092} 2056}
2093 2057
2094static int mlx5_set_bsf(struct ib_mr *sig_mr, 2058static int mlx5_set_bsf(struct ib_mr *sig_mr,
@@ -2099,45 +2063,49 @@ static int mlx5_set_bsf(struct ib_mr *sig_mr,
2099 struct mlx5_bsf_basic *basic = &bsf->basic; 2063 struct mlx5_bsf_basic *basic = &bsf->basic;
2100 struct ib_sig_domain *mem = &sig_attrs->mem; 2064 struct ib_sig_domain *mem = &sig_attrs->mem;
2101 struct ib_sig_domain *wire = &sig_attrs->wire; 2065 struct ib_sig_domain *wire = &sig_attrs->wire;
2102 int ret, selector;
2103 2066
2104 memset(bsf, 0, sizeof(*bsf)); 2067 memset(bsf, 0, sizeof(*bsf));
2068
2069 /* Basic + Extended + Inline */
2070 basic->bsf_size_sbs = 1 << 7;
2071 /* Input domain check byte mask */
2072 basic->check_byte_mask = sig_attrs->check_mask;
2073 basic->raw_data_size = cpu_to_be32(data_size);
2074
2075 /* Memory domain */
2105 switch (sig_attrs->mem.sig_type) { 2076 switch (sig_attrs->mem.sig_type) {
2077 case IB_SIG_TYPE_NONE:
2078 break;
2106 case IB_SIG_TYPE_T10_DIF: 2079 case IB_SIG_TYPE_T10_DIF:
2107 if (sig_attrs->wire.sig_type != IB_SIG_TYPE_T10_DIF) 2080 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval);
2108 return -EINVAL; 2081 basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx);
2082 mlx5_fill_inl_bsf(mem, &bsf->m_inl);
2083 break;
2084 default:
2085 return -EINVAL;
2086 }
2109 2087
2110 /* Input domain check byte mask */ 2088 /* Wire domain */
2111 basic->check_byte_mask = sig_attrs->check_mask; 2089 switch (sig_attrs->wire.sig_type) {
2090 case IB_SIG_TYPE_NONE:
2091 break;
2092 case IB_SIG_TYPE_T10_DIF:
2112 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval && 2093 if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval &&
2113 mem->sig.dif.type == wire->sig.dif.type) { 2094 mem->sig_type == wire->sig_type) {
2114 /* Same block structure */ 2095 /* Same block structure */
2115 basic->bsf_size_sbs = 1 << 4; 2096 basic->bsf_size_sbs |= 1 << 4;
2116 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type) 2097 if (mem->sig.dif.bg_type == wire->sig.dif.bg_type)
2117 basic->wire.copy_byte_mask |= 0xc0; 2098 basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK;
2118 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag) 2099 if (mem->sig.dif.app_tag == wire->sig.dif.app_tag)
2119 basic->wire.copy_byte_mask |= 0x30; 2100 basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK;
2120 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag) 2101 if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag)
2121 basic->wire.copy_byte_mask |= 0x0f; 2102 basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK;
2122 } else 2103 } else
2123 basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval); 2104 basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval);
2124 2105
2125 basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval); 2106 basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx);
2126 basic->raw_data_size = cpu_to_be32(data_size); 2107 mlx5_fill_inl_bsf(wire, &bsf->w_inl);
2127
2128 ret = format_selector(sig_attrs, mem, &selector);
2129 if (ret)
2130 return -EINVAL;
2131 basic->m_bfs_psv = cpu_to_be32(selector << 24 |
2132 msig->psv_memory.psv_idx);
2133
2134 ret = format_selector(sig_attrs, wire, &selector);
2135 if (ret)
2136 return -EINVAL;
2137 basic->w_bfs_psv = cpu_to_be32(selector << 24 |
2138 msig->psv_wire.psv_idx);
2139 break; 2108 break;
2140
2141 default: 2109 default:
2142 return -EINVAL; 2110 return -EINVAL;
2143 } 2111 }
@@ -2336,20 +2304,21 @@ static int set_psv_wr(struct ib_sig_domain *domain,
2336 memset(psv_seg, 0, sizeof(*psv_seg)); 2304 memset(psv_seg, 0, sizeof(*psv_seg));
2337 psv_seg->psv_num = cpu_to_be32(psv_idx); 2305 psv_seg->psv_num = cpu_to_be32(psv_idx);
2338 switch (domain->sig_type) { 2306 switch (domain->sig_type) {
2307 case IB_SIG_TYPE_NONE:
2308 break;
2339 case IB_SIG_TYPE_T10_DIF: 2309 case IB_SIG_TYPE_T10_DIF:
2340 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 | 2310 psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 |
2341 domain->sig.dif.app_tag); 2311 domain->sig.dif.app_tag);
2342 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag); 2312 psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag);
2343
2344 *seg += sizeof(*psv_seg);
2345 *size += sizeof(*psv_seg) / 16;
2346 break; 2313 break;
2347
2348 default: 2314 default:
2349 pr_err("Bad signature type given.\n"); 2315 pr_err("Bad signature type given.\n");
2350 return 1; 2316 return 1;
2351 } 2317 }
2352 2318
2319 *seg += sizeof(*psv_seg);
2320 *size += sizeof(*psv_seg) / 16;
2321
2353 return 0; 2322 return 0;
2354} 2323}
2355 2324
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
index dd35ae558ae1..638bff1ffc6c 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c
@@ -348,11 +348,6 @@ static void *ocrdma_init_emb_mqe(u8 opcode, u32 cmd_len)
348 return mqe; 348 return mqe;
349} 349}
350 350
351static void *ocrdma_alloc_mqe(void)
352{
353 return kzalloc(sizeof(struct ocrdma_mqe), GFP_KERNEL);
354}
355
356static void ocrdma_free_q(struct ocrdma_dev *dev, struct ocrdma_queue_info *q) 351static void ocrdma_free_q(struct ocrdma_dev *dev, struct ocrdma_queue_info *q)
357{ 352{
358 dma_free_coherent(&dev->nic_info.pdev->dev, q->size, q->va, q->dma); 353 dma_free_coherent(&dev->nic_info.pdev->dev, q->size, q->va, q->dma);
@@ -566,8 +561,8 @@ static int ocrdma_mbx_create_mq(struct ocrdma_dev *dev,
566 cmd->cqid_pages |= (cq->id << OCRDMA_CREATE_MQ_CQ_ID_SHIFT); 561 cmd->cqid_pages |= (cq->id << OCRDMA_CREATE_MQ_CQ_ID_SHIFT);
567 cmd->async_cqid_valid = OCRDMA_CREATE_MQ_ASYNC_CQ_VALID; 562 cmd->async_cqid_valid = OCRDMA_CREATE_MQ_ASYNC_CQ_VALID;
568 563
569 cmd->async_event_bitmap = Bit(OCRDMA_ASYNC_GRP5_EVE_CODE); 564 cmd->async_event_bitmap = BIT(OCRDMA_ASYNC_GRP5_EVE_CODE);
570 cmd->async_event_bitmap |= Bit(OCRDMA_ASYNC_RDMA_EVE_CODE); 565 cmd->async_event_bitmap |= BIT(OCRDMA_ASYNC_RDMA_EVE_CODE);
571 566
572 cmd->async_cqid_ringsize = cq->id; 567 cmd->async_cqid_ringsize = cq->id;
573 cmd->async_cqid_ringsize |= (ocrdma_encoded_q_len(mq->len) << 568 cmd->async_cqid_ringsize |= (ocrdma_encoded_q_len(mq->len) <<
@@ -1189,10 +1184,10 @@ int ocrdma_mbx_rdma_stats(struct ocrdma_dev *dev, bool reset)
1189{ 1184{
1190 struct ocrdma_rdma_stats_req *req = dev->stats_mem.va; 1185 struct ocrdma_rdma_stats_req *req = dev->stats_mem.va;
1191 struct ocrdma_mqe *mqe = &dev->stats_mem.mqe; 1186 struct ocrdma_mqe *mqe = &dev->stats_mem.mqe;
1192 struct ocrdma_rdma_stats_resp *old_stats = NULL; 1187 struct ocrdma_rdma_stats_resp *old_stats;
1193 int status; 1188 int status;
1194 1189
1195 old_stats = kzalloc(sizeof(*old_stats), GFP_KERNEL); 1190 old_stats = kmalloc(sizeof(*old_stats), GFP_KERNEL);
1196 if (old_stats == NULL) 1191 if (old_stats == NULL)
1197 return -ENOMEM; 1192 return -ENOMEM;
1198 1193
@@ -1235,10 +1230,9 @@ static int ocrdma_mbx_get_ctrl_attribs(struct ocrdma_dev *dev)
1235 struct ocrdma_get_ctrl_attribs_rsp *ctrl_attr_rsp; 1230 struct ocrdma_get_ctrl_attribs_rsp *ctrl_attr_rsp;
1236 struct mgmt_hba_attribs *hba_attribs; 1231 struct mgmt_hba_attribs *hba_attribs;
1237 1232
1238 mqe = ocrdma_alloc_mqe(); 1233 mqe = kzalloc(sizeof(struct ocrdma_mqe), GFP_KERNEL);
1239 if (!mqe) 1234 if (!mqe)
1240 return status; 1235 return status;
1241 memset(mqe, 0, sizeof(*mqe));
1242 1236
1243 dma.size = sizeof(struct ocrdma_get_ctrl_attribs_rsp); 1237 dma.size = sizeof(struct ocrdma_get_ctrl_attribs_rsp);
1244 dma.va = dma_alloc_coherent(&dev->nic_info.pdev->dev, 1238 dma.va = dma_alloc_coherent(&dev->nic_info.pdev->dev,
@@ -2279,7 +2273,8 @@ mbx_err:
2279 2273
2280static int ocrdma_set_av_params(struct ocrdma_qp *qp, 2274static int ocrdma_set_av_params(struct ocrdma_qp *qp,
2281 struct ocrdma_modify_qp *cmd, 2275 struct ocrdma_modify_qp *cmd,
2282 struct ib_qp_attr *attrs) 2276 struct ib_qp_attr *attrs,
2277 int attr_mask)
2283{ 2278{
2284 int status; 2279 int status;
2285 struct ib_ah_attr *ah_attr = &attrs->ah_attr; 2280 struct ib_ah_attr *ah_attr = &attrs->ah_attr;
@@ -2319,8 +2314,8 @@ static int ocrdma_set_av_params(struct ocrdma_qp *qp,
2319 ocrdma_cpu_to_le32(&cmd->params.dgid[0], sizeof(cmd->params.dgid)); 2314 ocrdma_cpu_to_le32(&cmd->params.dgid[0], sizeof(cmd->params.dgid));
2320 ocrdma_cpu_to_le32(&cmd->params.sgid[0], sizeof(cmd->params.sgid)); 2315 ocrdma_cpu_to_le32(&cmd->params.sgid[0], sizeof(cmd->params.sgid));
2321 cmd->params.vlan_dmac_b4_to_b5 = mac_addr[4] | (mac_addr[5] << 8); 2316 cmd->params.vlan_dmac_b4_to_b5 = mac_addr[4] | (mac_addr[5] << 8);
2322 vlan_id = ah_attr->vlan_id; 2317 if (attr_mask & IB_QP_VID) {
2323 if (vlan_id && (vlan_id < 0x1000)) { 2318 vlan_id = attrs->vlan_id;
2324 cmd->params.vlan_dmac_b4_to_b5 |= 2319 cmd->params.vlan_dmac_b4_to_b5 |=
2325 vlan_id << OCRDMA_QP_PARAMS_VLAN_SHIFT; 2320 vlan_id << OCRDMA_QP_PARAMS_VLAN_SHIFT;
2326 cmd->flags |= OCRDMA_QP_PARA_VLAN_EN_VALID; 2321 cmd->flags |= OCRDMA_QP_PARA_VLAN_EN_VALID;
@@ -2347,7 +2342,7 @@ static int ocrdma_set_qp_params(struct ocrdma_qp *qp,
2347 cmd->flags |= OCRDMA_QP_PARA_QKEY_VALID; 2342 cmd->flags |= OCRDMA_QP_PARA_QKEY_VALID;
2348 } 2343 }
2349 if (attr_mask & IB_QP_AV) { 2344 if (attr_mask & IB_QP_AV) {
2350 status = ocrdma_set_av_params(qp, cmd, attrs); 2345 status = ocrdma_set_av_params(qp, cmd, attrs, attr_mask);
2351 if (status) 2346 if (status)
2352 return status; 2347 return status;
2353 } else if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_UD) { 2348 } else if (qp->qp_type == IB_QPT_GSI || qp->qp_type == IB_QPT_UD) {
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
index 256a06bc0b68..b0b2257b8e04 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c
@@ -388,6 +388,15 @@ static void ocrdma_remove_sysfiles(struct ocrdma_dev *dev)
388 device_remove_file(&dev->ibdev.dev, ocrdma_attributes[i]); 388 device_remove_file(&dev->ibdev.dev, ocrdma_attributes[i]);
389} 389}
390 390
391static void ocrdma_add_default_sgid(struct ocrdma_dev *dev)
392{
393 /* GID Index 0 - Invariant manufacturer-assigned EUI-64 */
394 union ib_gid *sgid = &dev->sgid_tbl[0];
395
396 sgid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
397 ocrdma_get_guid(dev, &sgid->raw[8]);
398}
399
391static void ocrdma_init_ipv4_gids(struct ocrdma_dev *dev, 400static void ocrdma_init_ipv4_gids(struct ocrdma_dev *dev,
392 struct net_device *net) 401 struct net_device *net)
393{ 402{
@@ -434,6 +443,7 @@ static void ocrdma_init_gid_table(struct ocrdma_dev *dev)
434 rdma_vlan_dev_real_dev(net_dev) : net_dev; 443 rdma_vlan_dev_real_dev(net_dev) : net_dev;
435 444
436 if (real_dev == dev->nic_info.netdev) { 445 if (real_dev == dev->nic_info.netdev) {
446 ocrdma_add_default_sgid(dev);
437 ocrdma_init_ipv4_gids(dev, net_dev); 447 ocrdma_init_ipv4_gids(dev, net_dev);
438 ocrdma_init_ipv6_gids(dev, net_dev); 448 ocrdma_init_ipv6_gids(dev, net_dev);
439 } 449 }
@@ -646,8 +656,10 @@ static int __init ocrdma_init_module(void)
646 return 0; 656 return 0;
647 657
648err_be_reg: 658err_be_reg:
659#if IS_ENABLED(CONFIG_IPV6)
649 ocrdma_unregister_inet6addr_notifier(); 660 ocrdma_unregister_inet6addr_notifier();
650err_notifier6: 661err_notifier6:
662#endif
651 ocrdma_unregister_inetaddr_notifier(); 663 ocrdma_unregister_inetaddr_notifier();
652 return status; 664 return status;
653} 665}
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
index 904989ec5eaa..4e036480c1a8 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_sli.h
@@ -28,8 +28,6 @@
28#ifndef __OCRDMA_SLI_H__ 28#ifndef __OCRDMA_SLI_H__
29#define __OCRDMA_SLI_H__ 29#define __OCRDMA_SLI_H__
30 30
31#define Bit(_b) (1 << (_b))
32
33enum { 31enum {
34 OCRDMA_ASIC_GEN_SKH_R = 0x04, 32 OCRDMA_ASIC_GEN_SKH_R = 0x04,
35 OCRDMA_ASIC_GEN_LANCER = 0x0B 33 OCRDMA_ASIC_GEN_LANCER = 0x0B
@@ -103,7 +101,7 @@ enum {
103 QTYPE_MCCQ = 3 101 QTYPE_MCCQ = 3
104}; 102};
105 103
106#define OCRDMA_MAX_SGID (8) 104#define OCRDMA_MAX_SGID 8
107 105
108#define OCRDMA_MAX_QP 2048 106#define OCRDMA_MAX_QP 2048
109#define OCRDMA_MAX_CQ 2048 107#define OCRDMA_MAX_CQ 2048
@@ -128,33 +126,33 @@ enum {
128#define OCRDMA_DB_CQ_RING_ID_EXT_MASK 0x0C00 /* bits 10-11 of qid at 12-11 */ 126#define OCRDMA_DB_CQ_RING_ID_EXT_MASK 0x0C00 /* bits 10-11 of qid at 12-11 */
129/* qid #2 msbits at 12-11 */ 127/* qid #2 msbits at 12-11 */
130#define OCRDMA_DB_CQ_RING_ID_EXT_MASK_SHIFT 0x1 128#define OCRDMA_DB_CQ_RING_ID_EXT_MASK_SHIFT 0x1
131#define OCRDMA_DB_CQ_NUM_POPPED_SHIFT (16) /* bits 16 - 28 */ 129#define OCRDMA_DB_CQ_NUM_POPPED_SHIFT 16 /* bits 16 - 28 */
132/* Rearm bit */ 130/* Rearm bit */
133#define OCRDMA_DB_CQ_REARM_SHIFT (29) /* bit 29 */ 131#define OCRDMA_DB_CQ_REARM_SHIFT 29 /* bit 29 */
134/* solicited bit */ 132/* solicited bit */
135#define OCRDMA_DB_CQ_SOLICIT_SHIFT (31) /* bit 31 */ 133#define OCRDMA_DB_CQ_SOLICIT_SHIFT 31 /* bit 31 */
136 134
137#define OCRDMA_EQ_ID_MASK 0x1FF /* bits 0 - 8 */ 135#define OCRDMA_EQ_ID_MASK 0x1FF /* bits 0 - 8 */
138#define OCRDMA_EQ_ID_EXT_MASK 0x3e00 /* bits 9-13 */ 136#define OCRDMA_EQ_ID_EXT_MASK 0x3e00 /* bits 9-13 */
139#define OCRDMA_EQ_ID_EXT_MASK_SHIFT (2) /* qid bits 9-13 at 11-15 */ 137#define OCRDMA_EQ_ID_EXT_MASK_SHIFT 2 /* qid bits 9-13 at 11-15 */
140 138
141/* Clear the interrupt for this eq */ 139/* Clear the interrupt for this eq */
142#define OCRDMA_EQ_CLR_SHIFT (9) /* bit 9 */ 140#define OCRDMA_EQ_CLR_SHIFT 9 /* bit 9 */
143/* Must be 1 */ 141/* Must be 1 */
144#define OCRDMA_EQ_TYPE_SHIFT (10) /* bit 10 */ 142#define OCRDMA_EQ_TYPE_SHIFT 10 /* bit 10 */
145/* Number of event entries processed */ 143/* Number of event entries processed */
146#define OCRDMA_NUM_EQE_SHIFT (16) /* bits 16 - 28 */ 144#define OCRDMA_NUM_EQE_SHIFT 16 /* bits 16 - 28 */
147/* Rearm bit */ 145/* Rearm bit */
148#define OCRDMA_REARM_SHIFT (29) /* bit 29 */ 146#define OCRDMA_REARM_SHIFT 29 /* bit 29 */
149 147
150#define OCRDMA_MQ_ID_MASK 0x7FF /* bits 0 - 10 */ 148#define OCRDMA_MQ_ID_MASK 0x7FF /* bits 0 - 10 */
151/* Number of entries posted */ 149/* Number of entries posted */
152#define OCRDMA_MQ_NUM_MQE_SHIFT (16) /* bits 16 - 29 */ 150#define OCRDMA_MQ_NUM_MQE_SHIFT 16 /* bits 16 - 29 */
153 151
154#define OCRDMA_MIN_HPAGE_SIZE (4096) 152#define OCRDMA_MIN_HPAGE_SIZE 4096
155 153
156#define OCRDMA_MIN_Q_PAGE_SIZE (4096) 154#define OCRDMA_MIN_Q_PAGE_SIZE 4096
157#define OCRDMA_MAX_Q_PAGES (8) 155#define OCRDMA_MAX_Q_PAGES 8
158 156
159#define OCRDMA_SLI_ASIC_ID_OFFSET 0x9C 157#define OCRDMA_SLI_ASIC_ID_OFFSET 0x9C
160#define OCRDMA_SLI_ASIC_REV_MASK 0x000000FF 158#define OCRDMA_SLI_ASIC_REV_MASK 0x000000FF
@@ -170,14 +168,14 @@ enum {
170# 6: 256K Bytes 168# 6: 256K Bytes
171# 7: 512K Bytes 169# 7: 512K Bytes
172*/ 170*/
173#define OCRDMA_MAX_Q_PAGE_SIZE_CNT (8) 171#define OCRDMA_MAX_Q_PAGE_SIZE_CNT 8
174#define OCRDMA_Q_PAGE_BASE_SIZE (OCRDMA_MIN_Q_PAGE_SIZE * OCRDMA_MAX_Q_PAGES) 172#define OCRDMA_Q_PAGE_BASE_SIZE (OCRDMA_MIN_Q_PAGE_SIZE * OCRDMA_MAX_Q_PAGES)
175 173
176#define MAX_OCRDMA_QP_PAGES (8) 174#define MAX_OCRDMA_QP_PAGES 8
177#define OCRDMA_MAX_WQE_MEM_SIZE (MAX_OCRDMA_QP_PAGES * OCRDMA_MIN_HQ_PAGE_SIZE) 175#define OCRDMA_MAX_WQE_MEM_SIZE (MAX_OCRDMA_QP_PAGES * OCRDMA_MIN_HQ_PAGE_SIZE)
178 176
179#define OCRDMA_CREATE_CQ_MAX_PAGES (4) 177#define OCRDMA_CREATE_CQ_MAX_PAGES 4
180#define OCRDMA_DPP_CQE_SIZE (4) 178#define OCRDMA_DPP_CQE_SIZE 4
181 179
182#define OCRDMA_GEN2_MAX_CQE 1024 180#define OCRDMA_GEN2_MAX_CQE 1024
183#define OCRDMA_GEN2_CQ_PAGE_SIZE 4096 181#define OCRDMA_GEN2_CQ_PAGE_SIZE 4096
@@ -238,7 +236,7 @@ struct ocrdma_mqe_sge {
238 236
239enum { 237enum {
240 OCRDMA_MQE_HDR_EMB_SHIFT = 0, 238 OCRDMA_MQE_HDR_EMB_SHIFT = 0,
241 OCRDMA_MQE_HDR_EMB_MASK = Bit(0), 239 OCRDMA_MQE_HDR_EMB_MASK = BIT(0),
242 OCRDMA_MQE_HDR_SGE_CNT_SHIFT = 3, 240 OCRDMA_MQE_HDR_SGE_CNT_SHIFT = 3,
243 OCRDMA_MQE_HDR_SGE_CNT_MASK = 0x1F << OCRDMA_MQE_HDR_SGE_CNT_SHIFT, 241 OCRDMA_MQE_HDR_SGE_CNT_MASK = 0x1F << OCRDMA_MQE_HDR_SGE_CNT_SHIFT,
244 OCRDMA_MQE_HDR_SPECIAL_SHIFT = 24, 242 OCRDMA_MQE_HDR_SPECIAL_SHIFT = 24,
@@ -292,7 +290,7 @@ struct ocrdma_pa {
292 u32 hi; 290 u32 hi;
293}; 291};
294 292
295#define MAX_OCRDMA_EQ_PAGES (8) 293#define MAX_OCRDMA_EQ_PAGES 8
296struct ocrdma_create_eq_req { 294struct ocrdma_create_eq_req {
297 struct ocrdma_mbx_hdr req; 295 struct ocrdma_mbx_hdr req;
298 u32 num_pages; 296 u32 num_pages;
@@ -304,7 +302,7 @@ struct ocrdma_create_eq_req {
304}; 302};
305 303
306enum { 304enum {
307 OCRDMA_CREATE_EQ_VALID = Bit(29), 305 OCRDMA_CREATE_EQ_VALID = BIT(29),
308 OCRDMA_CREATE_EQ_CNT_SHIFT = 26, 306 OCRDMA_CREATE_EQ_CNT_SHIFT = 26,
309 OCRDMA_CREATE_CQ_DELAY_SHIFT = 13, 307 OCRDMA_CREATE_CQ_DELAY_SHIFT = 13,
310}; 308};
@@ -314,7 +312,7 @@ struct ocrdma_create_eq_rsp {
314 u32 vector_eqid; 312 u32 vector_eqid;
315}; 313};
316 314
317#define OCRDMA_EQ_MINOR_OTHER (0x1) 315#define OCRDMA_EQ_MINOR_OTHER 0x1
318 316
319enum { 317enum {
320 OCRDMA_MCQE_STATUS_SHIFT = 0, 318 OCRDMA_MCQE_STATUS_SHIFT = 0,
@@ -322,13 +320,13 @@ enum {
322 OCRDMA_MCQE_ESTATUS_SHIFT = 16, 320 OCRDMA_MCQE_ESTATUS_SHIFT = 16,
323 OCRDMA_MCQE_ESTATUS_MASK = 0xFFFF << OCRDMA_MCQE_ESTATUS_SHIFT, 321 OCRDMA_MCQE_ESTATUS_MASK = 0xFFFF << OCRDMA_MCQE_ESTATUS_SHIFT,
324 OCRDMA_MCQE_CONS_SHIFT = 27, 322 OCRDMA_MCQE_CONS_SHIFT = 27,
325 OCRDMA_MCQE_CONS_MASK = Bit(27), 323 OCRDMA_MCQE_CONS_MASK = BIT(27),
326 OCRDMA_MCQE_CMPL_SHIFT = 28, 324 OCRDMA_MCQE_CMPL_SHIFT = 28,
327 OCRDMA_MCQE_CMPL_MASK = Bit(28), 325 OCRDMA_MCQE_CMPL_MASK = BIT(28),
328 OCRDMA_MCQE_AE_SHIFT = 30, 326 OCRDMA_MCQE_AE_SHIFT = 30,
329 OCRDMA_MCQE_AE_MASK = Bit(30), 327 OCRDMA_MCQE_AE_MASK = BIT(30),
330 OCRDMA_MCQE_VALID_SHIFT = 31, 328 OCRDMA_MCQE_VALID_SHIFT = 31,
331 OCRDMA_MCQE_VALID_MASK = Bit(31) 329 OCRDMA_MCQE_VALID_MASK = BIT(31)
332}; 330};
333 331
334struct ocrdma_mcqe { 332struct ocrdma_mcqe {
@@ -339,13 +337,13 @@ struct ocrdma_mcqe {
339}; 337};
340 338
341enum { 339enum {
342 OCRDMA_AE_MCQE_QPVALID = Bit(31), 340 OCRDMA_AE_MCQE_QPVALID = BIT(31),
343 OCRDMA_AE_MCQE_QPID_MASK = 0xFFFF, 341 OCRDMA_AE_MCQE_QPID_MASK = 0xFFFF,
344 342
345 OCRDMA_AE_MCQE_CQVALID = Bit(31), 343 OCRDMA_AE_MCQE_CQVALID = BIT(31),
346 OCRDMA_AE_MCQE_CQID_MASK = 0xFFFF, 344 OCRDMA_AE_MCQE_CQID_MASK = 0xFFFF,
347 OCRDMA_AE_MCQE_VALID = Bit(31), 345 OCRDMA_AE_MCQE_VALID = BIT(31),
348 OCRDMA_AE_MCQE_AE = Bit(30), 346 OCRDMA_AE_MCQE_AE = BIT(30),
349 OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT = 16, 347 OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT = 16,
350 OCRDMA_AE_MCQE_EVENT_TYPE_MASK = 348 OCRDMA_AE_MCQE_EVENT_TYPE_MASK =
351 0xFF << OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT, 349 0xFF << OCRDMA_AE_MCQE_EVENT_TYPE_SHIFT,
@@ -386,9 +384,9 @@ enum {
386 OCRDMA_AE_MPA_MCQE_EVENT_TYPE_MASK = 0xFF << 384 OCRDMA_AE_MPA_MCQE_EVENT_TYPE_MASK = 0xFF <<
387 OCRDMA_AE_MPA_MCQE_EVENT_TYPE_SHIFT, 385 OCRDMA_AE_MPA_MCQE_EVENT_TYPE_SHIFT,
388 OCRDMA_AE_MPA_MCQE_EVENT_AE_SHIFT = 30, 386 OCRDMA_AE_MPA_MCQE_EVENT_AE_SHIFT = 30,
389 OCRDMA_AE_MPA_MCQE_EVENT_AE_MASK = Bit(30), 387 OCRDMA_AE_MPA_MCQE_EVENT_AE_MASK = BIT(30),
390 OCRDMA_AE_MPA_MCQE_EVENT_VALID_SHIFT = 31, 388 OCRDMA_AE_MPA_MCQE_EVENT_VALID_SHIFT = 31,
391 OCRDMA_AE_MPA_MCQE_EVENT_VALID_MASK = Bit(31) 389 OCRDMA_AE_MPA_MCQE_EVENT_VALID_MASK = BIT(31)
392}; 390};
393 391
394struct ocrdma_ae_mpa_mcqe { 392struct ocrdma_ae_mpa_mcqe {
@@ -412,9 +410,9 @@ enum {
412 OCRDMA_AE_QP_MCQE_EVENT_TYPE_MASK = 0xFF << 410 OCRDMA_AE_QP_MCQE_EVENT_TYPE_MASK = 0xFF <<
413 OCRDMA_AE_QP_MCQE_EVENT_TYPE_SHIFT, 411 OCRDMA_AE_QP_MCQE_EVENT_TYPE_SHIFT,
414 OCRDMA_AE_QP_MCQE_EVENT_AE_SHIFT = 30, 412 OCRDMA_AE_QP_MCQE_EVENT_AE_SHIFT = 30,
415 OCRDMA_AE_QP_MCQE_EVENT_AE_MASK = Bit(30), 413 OCRDMA_AE_QP_MCQE_EVENT_AE_MASK = BIT(30),
416 OCRDMA_AE_QP_MCQE_EVENT_VALID_SHIFT = 31, 414 OCRDMA_AE_QP_MCQE_EVENT_VALID_SHIFT = 31,
417 OCRDMA_AE_QP_MCQE_EVENT_VALID_MASK = Bit(31) 415 OCRDMA_AE_QP_MCQE_EVENT_VALID_MASK = BIT(31)
418}; 416};
419 417
420struct ocrdma_ae_qp_mcqe { 418struct ocrdma_ae_qp_mcqe {
@@ -449,9 +447,9 @@ enum OCRDMA_ASYNC_EVENT_TYPE {
449/* mailbox command request and responses */ 447/* mailbox command request and responses */
450enum { 448enum {
451 OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_SHIFT = 2, 449 OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_SHIFT = 2,
452 OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_MASK = Bit(2), 450 OCRDMA_MBX_QUERY_CFG_CQ_OVERFLOW_MASK = BIT(2),
453 OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_SHIFT = 3, 451 OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_SHIFT = 3,
454 OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_MASK = Bit(3), 452 OCRDMA_MBX_QUERY_CFG_SRQ_SUPPORTED_MASK = BIT(3),
455 OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT = 8, 453 OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT = 8,
456 OCRDMA_MBX_QUERY_CFG_MAX_QP_MASK = 0xFFFFFF << 454 OCRDMA_MBX_QUERY_CFG_MAX_QP_MASK = 0xFFFFFF <<
457 OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT, 455 OCRDMA_MBX_QUERY_CFG_MAX_QP_SHIFT,
@@ -672,9 +670,9 @@ enum {
672 OCRDMA_CREATE_CQ_PAGE_SIZE_MASK = 0xFF, 670 OCRDMA_CREATE_CQ_PAGE_SIZE_MASK = 0xFF,
673 671
674 OCRDMA_CREATE_CQ_COALESCWM_SHIFT = 12, 672 OCRDMA_CREATE_CQ_COALESCWM_SHIFT = 12,
675 OCRDMA_CREATE_CQ_COALESCWM_MASK = Bit(13) | Bit(12), 673 OCRDMA_CREATE_CQ_COALESCWM_MASK = BIT(13) | BIT(12),
676 OCRDMA_CREATE_CQ_FLAGS_NODELAY = Bit(14), 674 OCRDMA_CREATE_CQ_FLAGS_NODELAY = BIT(14),
677 OCRDMA_CREATE_CQ_FLAGS_AUTO_VALID = Bit(15), 675 OCRDMA_CREATE_CQ_FLAGS_AUTO_VALID = BIT(15),
678 676
679 OCRDMA_CREATE_CQ_EQ_ID_MASK = 0xFFFF, 677 OCRDMA_CREATE_CQ_EQ_ID_MASK = 0xFFFF,
680 OCRDMA_CREATE_CQ_CQE_COUNT_MASK = 0xFFFF 678 OCRDMA_CREATE_CQ_CQE_COUNT_MASK = 0xFFFF
@@ -687,8 +685,8 @@ enum {
687 OCRDMA_CREATE_CQ_EQID_SHIFT = 22, 685 OCRDMA_CREATE_CQ_EQID_SHIFT = 22,
688 686
689 OCRDMA_CREATE_CQ_CNT_SHIFT = 27, 687 OCRDMA_CREATE_CQ_CNT_SHIFT = 27,
690 OCRDMA_CREATE_CQ_FLAGS_VALID = Bit(29), 688 OCRDMA_CREATE_CQ_FLAGS_VALID = BIT(29),
691 OCRDMA_CREATE_CQ_FLAGS_EVENTABLE = Bit(31), 689 OCRDMA_CREATE_CQ_FLAGS_EVENTABLE = BIT(31),
692 OCRDMA_CREATE_CQ_DEF_FLAGS = OCRDMA_CREATE_CQ_FLAGS_VALID | 690 OCRDMA_CREATE_CQ_DEF_FLAGS = OCRDMA_CREATE_CQ_FLAGS_VALID |
693 OCRDMA_CREATE_CQ_FLAGS_EVENTABLE | 691 OCRDMA_CREATE_CQ_FLAGS_EVENTABLE |
694 OCRDMA_CREATE_CQ_FLAGS_NODELAY 692 OCRDMA_CREATE_CQ_FLAGS_NODELAY
@@ -731,8 +729,8 @@ enum {
731 OCRDMA_CREATE_MQ_V0_CQ_ID_SHIFT = 22, 729 OCRDMA_CREATE_MQ_V0_CQ_ID_SHIFT = 22,
732 OCRDMA_CREATE_MQ_CQ_ID_SHIFT = 16, 730 OCRDMA_CREATE_MQ_CQ_ID_SHIFT = 16,
733 OCRDMA_CREATE_MQ_RING_SIZE_SHIFT = 16, 731 OCRDMA_CREATE_MQ_RING_SIZE_SHIFT = 16,
734 OCRDMA_CREATE_MQ_VALID = Bit(31), 732 OCRDMA_CREATE_MQ_VALID = BIT(31),
735 OCRDMA_CREATE_MQ_ASYNC_CQ_VALID = Bit(0) 733 OCRDMA_CREATE_MQ_ASYNC_CQ_VALID = BIT(0)
736}; 734};
737 735
738struct ocrdma_create_mq_req { 736struct ocrdma_create_mq_req {
@@ -783,7 +781,7 @@ enum {
783 OCRDMA_CREATE_QP_REQ_SQ_PAGE_SIZE_SHIFT = 16, 781 OCRDMA_CREATE_QP_REQ_SQ_PAGE_SIZE_SHIFT = 16,
784 OCRDMA_CREATE_QP_REQ_RQ_PAGE_SIZE_SHIFT = 19, 782 OCRDMA_CREATE_QP_REQ_RQ_PAGE_SIZE_SHIFT = 19,
785 OCRDMA_CREATE_QP_REQ_QPT_SHIFT = 29, 783 OCRDMA_CREATE_QP_REQ_QPT_SHIFT = 29,
786 OCRDMA_CREATE_QP_REQ_QPT_MASK = Bit(31) | Bit(30) | Bit(29), 784 OCRDMA_CREATE_QP_REQ_QPT_MASK = BIT(31) | BIT(30) | BIT(29),
787 785
788 OCRDMA_CREATE_QP_REQ_MAX_RQE_SHIFT = 0, 786 OCRDMA_CREATE_QP_REQ_MAX_RQE_SHIFT = 0,
789 OCRDMA_CREATE_QP_REQ_MAX_RQE_MASK = 0xFFFF, 787 OCRDMA_CREATE_QP_REQ_MAX_RQE_MASK = 0xFFFF,
@@ -798,23 +796,23 @@ enum {
798 OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_SHIFT, 796 OCRDMA_CREATE_QP_REQ_MAX_SGE_SEND_SHIFT,
799 797
800 OCRDMA_CREATE_QP_REQ_FMR_EN_SHIFT = 0, 798 OCRDMA_CREATE_QP_REQ_FMR_EN_SHIFT = 0,
801 OCRDMA_CREATE_QP_REQ_FMR_EN_MASK = Bit(0), 799 OCRDMA_CREATE_QP_REQ_FMR_EN_MASK = BIT(0),
802 OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_SHIFT = 1, 800 OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_SHIFT = 1,
803 OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_MASK = Bit(1), 801 OCRDMA_CREATE_QP_REQ_ZERO_LKEYEN_MASK = BIT(1),
804 OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_SHIFT = 2, 802 OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_SHIFT = 2,
805 OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_MASK = Bit(2), 803 OCRDMA_CREATE_QP_REQ_BIND_MEMWIN_MASK = BIT(2),
806 OCRDMA_CREATE_QP_REQ_INB_WREN_SHIFT = 3, 804 OCRDMA_CREATE_QP_REQ_INB_WREN_SHIFT = 3,
807 OCRDMA_CREATE_QP_REQ_INB_WREN_MASK = Bit(3), 805 OCRDMA_CREATE_QP_REQ_INB_WREN_MASK = BIT(3),
808 OCRDMA_CREATE_QP_REQ_INB_RDEN_SHIFT = 4, 806 OCRDMA_CREATE_QP_REQ_INB_RDEN_SHIFT = 4,
809 OCRDMA_CREATE_QP_REQ_INB_RDEN_MASK = Bit(4), 807 OCRDMA_CREATE_QP_REQ_INB_RDEN_MASK = BIT(4),
810 OCRDMA_CREATE_QP_REQ_USE_SRQ_SHIFT = 5, 808 OCRDMA_CREATE_QP_REQ_USE_SRQ_SHIFT = 5,
811 OCRDMA_CREATE_QP_REQ_USE_SRQ_MASK = Bit(5), 809 OCRDMA_CREATE_QP_REQ_USE_SRQ_MASK = BIT(5),
812 OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_SHIFT = 6, 810 OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_SHIFT = 6,
813 OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_MASK = Bit(6), 811 OCRDMA_CREATE_QP_REQ_ENABLE_RPIR_MASK = BIT(6),
814 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_SHIFT = 7, 812 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_SHIFT = 7,
815 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_MASK = Bit(7), 813 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_MASK = BIT(7),
816 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_SHIFT = 8, 814 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_SHIFT = 8,
817 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_MASK = Bit(8), 815 OCRDMA_CREATE_QP_REQ_ENABLE_DPP_CQ_MASK = BIT(8),
818 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT = 16, 816 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT = 16,
819 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_MASK = 0xFFFF << 817 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_MASK = 0xFFFF <<
820 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT, 818 OCRDMA_CREATE_QP_REQ_MAX_SGE_RECV_SHIFT,
@@ -927,7 +925,7 @@ enum {
927 OCRDMA_CREATE_QP_RSP_SQ_ID_MASK = 0xFFFF << 925 OCRDMA_CREATE_QP_RSP_SQ_ID_MASK = 0xFFFF <<
928 OCRDMA_CREATE_QP_RSP_SQ_ID_SHIFT, 926 OCRDMA_CREATE_QP_RSP_SQ_ID_SHIFT,
929 927
930 OCRDMA_CREATE_QP_RSP_DPP_ENABLED_MASK = Bit(0), 928 OCRDMA_CREATE_QP_RSP_DPP_ENABLED_MASK = BIT(0),
931 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT = 1, 929 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT = 1,
932 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_MASK = 0x7FFF << 930 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_MASK = 0x7FFF <<
933 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT, 931 OCRDMA_CREATE_QP_RSP_DPP_PAGE_OFFSET_SHIFT,
@@ -964,38 +962,38 @@ enum {
964 OCRDMA_MODIFY_QP_ID_SHIFT = 0, 962 OCRDMA_MODIFY_QP_ID_SHIFT = 0,
965 OCRDMA_MODIFY_QP_ID_MASK = 0xFFFF, 963 OCRDMA_MODIFY_QP_ID_MASK = 0xFFFF,
966 964
967 OCRDMA_QP_PARA_QPS_VALID = Bit(0), 965 OCRDMA_QP_PARA_QPS_VALID = BIT(0),
968 OCRDMA_QP_PARA_SQD_ASYNC_VALID = Bit(1), 966 OCRDMA_QP_PARA_SQD_ASYNC_VALID = BIT(1),
969 OCRDMA_QP_PARA_PKEY_VALID = Bit(2), 967 OCRDMA_QP_PARA_PKEY_VALID = BIT(2),
970 OCRDMA_QP_PARA_QKEY_VALID = Bit(3), 968 OCRDMA_QP_PARA_QKEY_VALID = BIT(3),
971 OCRDMA_QP_PARA_PMTU_VALID = Bit(4), 969 OCRDMA_QP_PARA_PMTU_VALID = BIT(4),
972 OCRDMA_QP_PARA_ACK_TO_VALID = Bit(5), 970 OCRDMA_QP_PARA_ACK_TO_VALID = BIT(5),
973 OCRDMA_QP_PARA_RETRY_CNT_VALID = Bit(6), 971 OCRDMA_QP_PARA_RETRY_CNT_VALID = BIT(6),
974 OCRDMA_QP_PARA_RRC_VALID = Bit(7), 972 OCRDMA_QP_PARA_RRC_VALID = BIT(7),
975 OCRDMA_QP_PARA_RQPSN_VALID = Bit(8), 973 OCRDMA_QP_PARA_RQPSN_VALID = BIT(8),
976 OCRDMA_QP_PARA_MAX_IRD_VALID = Bit(9), 974 OCRDMA_QP_PARA_MAX_IRD_VALID = BIT(9),
977 OCRDMA_QP_PARA_MAX_ORD_VALID = Bit(10), 975 OCRDMA_QP_PARA_MAX_ORD_VALID = BIT(10),
978 OCRDMA_QP_PARA_RNT_VALID = Bit(11), 976 OCRDMA_QP_PARA_RNT_VALID = BIT(11),
979 OCRDMA_QP_PARA_SQPSN_VALID = Bit(12), 977 OCRDMA_QP_PARA_SQPSN_VALID = BIT(12),
980 OCRDMA_QP_PARA_DST_QPN_VALID = Bit(13), 978 OCRDMA_QP_PARA_DST_QPN_VALID = BIT(13),
981 OCRDMA_QP_PARA_MAX_WQE_VALID = Bit(14), 979 OCRDMA_QP_PARA_MAX_WQE_VALID = BIT(14),
982 OCRDMA_QP_PARA_MAX_RQE_VALID = Bit(15), 980 OCRDMA_QP_PARA_MAX_RQE_VALID = BIT(15),
983 OCRDMA_QP_PARA_SGE_SEND_VALID = Bit(16), 981 OCRDMA_QP_PARA_SGE_SEND_VALID = BIT(16),
984 OCRDMA_QP_PARA_SGE_RECV_VALID = Bit(17), 982 OCRDMA_QP_PARA_SGE_RECV_VALID = BIT(17),
985 OCRDMA_QP_PARA_SGE_WR_VALID = Bit(18), 983 OCRDMA_QP_PARA_SGE_WR_VALID = BIT(18),
986 OCRDMA_QP_PARA_INB_RDEN_VALID = Bit(19), 984 OCRDMA_QP_PARA_INB_RDEN_VALID = BIT(19),
987 OCRDMA_QP_PARA_INB_WREN_VALID = Bit(20), 985 OCRDMA_QP_PARA_INB_WREN_VALID = BIT(20),
988 OCRDMA_QP_PARA_FLOW_LBL_VALID = Bit(21), 986 OCRDMA_QP_PARA_FLOW_LBL_VALID = BIT(21),
989 OCRDMA_QP_PARA_BIND_EN_VALID = Bit(22), 987 OCRDMA_QP_PARA_BIND_EN_VALID = BIT(22),
990 OCRDMA_QP_PARA_ZLKEY_EN_VALID = Bit(23), 988 OCRDMA_QP_PARA_ZLKEY_EN_VALID = BIT(23),
991 OCRDMA_QP_PARA_FMR_EN_VALID = Bit(24), 989 OCRDMA_QP_PARA_FMR_EN_VALID = BIT(24),
992 OCRDMA_QP_PARA_INBAT_EN_VALID = Bit(25), 990 OCRDMA_QP_PARA_INBAT_EN_VALID = BIT(25),
993 OCRDMA_QP_PARA_VLAN_EN_VALID = Bit(26), 991 OCRDMA_QP_PARA_VLAN_EN_VALID = BIT(26),
994 992
995 OCRDMA_MODIFY_QP_FLAGS_RD = Bit(0), 993 OCRDMA_MODIFY_QP_FLAGS_RD = BIT(0),
996 OCRDMA_MODIFY_QP_FLAGS_WR = Bit(1), 994 OCRDMA_MODIFY_QP_FLAGS_WR = BIT(1),
997 OCRDMA_MODIFY_QP_FLAGS_SEND = Bit(2), 995 OCRDMA_MODIFY_QP_FLAGS_SEND = BIT(2),
998 OCRDMA_MODIFY_QP_FLAGS_ATOMIC = Bit(3) 996 OCRDMA_MODIFY_QP_FLAGS_ATOMIC = BIT(3)
999}; 997};
1000 998
1001enum { 999enum {
@@ -1014,15 +1012,15 @@ enum {
1014 OCRDMA_QP_PARAMS_MAX_SGE_SEND_MASK = 0xFFFF << 1012 OCRDMA_QP_PARAMS_MAX_SGE_SEND_MASK = 0xFFFF <<
1015 OCRDMA_QP_PARAMS_MAX_SGE_SEND_SHIFT, 1013 OCRDMA_QP_PARAMS_MAX_SGE_SEND_SHIFT,
1016 1014
1017 OCRDMA_QP_PARAMS_FLAGS_FMR_EN = Bit(0), 1015 OCRDMA_QP_PARAMS_FLAGS_FMR_EN = BIT(0),
1018 OCRDMA_QP_PARAMS_FLAGS_LKEY_0_EN = Bit(1), 1016 OCRDMA_QP_PARAMS_FLAGS_LKEY_0_EN = BIT(1),
1019 OCRDMA_QP_PARAMS_FLAGS_BIND_MW_EN = Bit(2), 1017 OCRDMA_QP_PARAMS_FLAGS_BIND_MW_EN = BIT(2),
1020 OCRDMA_QP_PARAMS_FLAGS_INBWR_EN = Bit(3), 1018 OCRDMA_QP_PARAMS_FLAGS_INBWR_EN = BIT(3),
1021 OCRDMA_QP_PARAMS_FLAGS_INBRD_EN = Bit(4), 1019 OCRDMA_QP_PARAMS_FLAGS_INBRD_EN = BIT(4),
1022 OCRDMA_QP_PARAMS_STATE_SHIFT = 5, 1020 OCRDMA_QP_PARAMS_STATE_SHIFT = 5,
1023 OCRDMA_QP_PARAMS_STATE_MASK = Bit(5) | Bit(6) | Bit(7), 1021 OCRDMA_QP_PARAMS_STATE_MASK = BIT(5) | BIT(6) | BIT(7),
1024 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC = Bit(8), 1022 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC = BIT(8),
1025 OCRDMA_QP_PARAMS_FLAGS_INB_ATEN = Bit(9), 1023 OCRDMA_QP_PARAMS_FLAGS_INB_ATEN = BIT(9),
1026 OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT = 16, 1024 OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT = 16,
1027 OCRDMA_QP_PARAMS_MAX_SGE_RECV_MASK = 0xFFFF << 1025 OCRDMA_QP_PARAMS_MAX_SGE_RECV_MASK = 0xFFFF <<
1028 OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT, 1026 OCRDMA_QP_PARAMS_MAX_SGE_RECV_SHIFT,
@@ -1277,7 +1275,7 @@ struct ocrdma_alloc_pd {
1277}; 1275};
1278 1276
1279enum { 1277enum {
1280 OCRDMA_ALLOC_PD_RSP_DPP = Bit(16), 1278 OCRDMA_ALLOC_PD_RSP_DPP = BIT(16),
1281 OCRDMA_ALLOC_PD_RSP_DPP_PAGE_SHIFT = 20, 1279 OCRDMA_ALLOC_PD_RSP_DPP_PAGE_SHIFT = 20,
1282 OCRDMA_ALLOC_PD_RSP_PDID_MASK = 0xFFFF, 1280 OCRDMA_ALLOC_PD_RSP_PDID_MASK = 0xFFFF,
1283}; 1281};
@@ -1309,18 +1307,18 @@ enum {
1309 OCRDMA_ALLOC_LKEY_PD_ID_MASK = 0xFFFF, 1307 OCRDMA_ALLOC_LKEY_PD_ID_MASK = 0xFFFF,
1310 1308
1311 OCRDMA_ALLOC_LKEY_ADDR_CHECK_SHIFT = 0, 1309 OCRDMA_ALLOC_LKEY_ADDR_CHECK_SHIFT = 0,
1312 OCRDMA_ALLOC_LKEY_ADDR_CHECK_MASK = Bit(0), 1310 OCRDMA_ALLOC_LKEY_ADDR_CHECK_MASK = BIT(0),
1313 OCRDMA_ALLOC_LKEY_FMR_SHIFT = 1, 1311 OCRDMA_ALLOC_LKEY_FMR_SHIFT = 1,
1314 OCRDMA_ALLOC_LKEY_FMR_MASK = Bit(1), 1312 OCRDMA_ALLOC_LKEY_FMR_MASK = BIT(1),
1315 OCRDMA_ALLOC_LKEY_REMOTE_INV_SHIFT = 2, 1313 OCRDMA_ALLOC_LKEY_REMOTE_INV_SHIFT = 2,
1316 OCRDMA_ALLOC_LKEY_REMOTE_INV_MASK = Bit(2), 1314 OCRDMA_ALLOC_LKEY_REMOTE_INV_MASK = BIT(2),
1317 OCRDMA_ALLOC_LKEY_REMOTE_WR_SHIFT = 3, 1315 OCRDMA_ALLOC_LKEY_REMOTE_WR_SHIFT = 3,
1318 OCRDMA_ALLOC_LKEY_REMOTE_WR_MASK = Bit(3), 1316 OCRDMA_ALLOC_LKEY_REMOTE_WR_MASK = BIT(3),
1319 OCRDMA_ALLOC_LKEY_REMOTE_RD_SHIFT = 4, 1317 OCRDMA_ALLOC_LKEY_REMOTE_RD_SHIFT = 4,
1320 OCRDMA_ALLOC_LKEY_REMOTE_RD_MASK = Bit(4), 1318 OCRDMA_ALLOC_LKEY_REMOTE_RD_MASK = BIT(4),
1321 OCRDMA_ALLOC_LKEY_LOCAL_WR_SHIFT = 5, 1319 OCRDMA_ALLOC_LKEY_LOCAL_WR_SHIFT = 5,
1322 OCRDMA_ALLOC_LKEY_LOCAL_WR_MASK = Bit(5), 1320 OCRDMA_ALLOC_LKEY_LOCAL_WR_MASK = BIT(5),
1323 OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_MASK = Bit(6), 1321 OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_MASK = BIT(6),
1324 OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_SHIFT = 6, 1322 OCRDMA_ALLOC_LKEY_REMOTE_ATOMIC_SHIFT = 6,
1325 OCRDMA_ALLOC_LKEY_PBL_SIZE_SHIFT = 16, 1323 OCRDMA_ALLOC_LKEY_PBL_SIZE_SHIFT = 16,
1326 OCRDMA_ALLOC_LKEY_PBL_SIZE_MASK = 0xFFFF << 1324 OCRDMA_ALLOC_LKEY_PBL_SIZE_MASK = 0xFFFF <<
@@ -1379,21 +1377,21 @@ enum {
1379 OCRDMA_REG_NSMR_HPAGE_SIZE_MASK = 0xFF << 1377 OCRDMA_REG_NSMR_HPAGE_SIZE_MASK = 0xFF <<
1380 OCRDMA_REG_NSMR_HPAGE_SIZE_SHIFT, 1378 OCRDMA_REG_NSMR_HPAGE_SIZE_SHIFT,
1381 OCRDMA_REG_NSMR_BIND_MEMWIN_SHIFT = 24, 1379 OCRDMA_REG_NSMR_BIND_MEMWIN_SHIFT = 24,
1382 OCRDMA_REG_NSMR_BIND_MEMWIN_MASK = Bit(24), 1380 OCRDMA_REG_NSMR_BIND_MEMWIN_MASK = BIT(24),
1383 OCRDMA_REG_NSMR_ZB_SHIFT = 25, 1381 OCRDMA_REG_NSMR_ZB_SHIFT = 25,
1384 OCRDMA_REG_NSMR_ZB_SHIFT_MASK = Bit(25), 1382 OCRDMA_REG_NSMR_ZB_SHIFT_MASK = BIT(25),
1385 OCRDMA_REG_NSMR_REMOTE_INV_SHIFT = 26, 1383 OCRDMA_REG_NSMR_REMOTE_INV_SHIFT = 26,
1386 OCRDMA_REG_NSMR_REMOTE_INV_MASK = Bit(26), 1384 OCRDMA_REG_NSMR_REMOTE_INV_MASK = BIT(26),
1387 OCRDMA_REG_NSMR_REMOTE_WR_SHIFT = 27, 1385 OCRDMA_REG_NSMR_REMOTE_WR_SHIFT = 27,
1388 OCRDMA_REG_NSMR_REMOTE_WR_MASK = Bit(27), 1386 OCRDMA_REG_NSMR_REMOTE_WR_MASK = BIT(27),
1389 OCRDMA_REG_NSMR_REMOTE_RD_SHIFT = 28, 1387 OCRDMA_REG_NSMR_REMOTE_RD_SHIFT = 28,
1390 OCRDMA_REG_NSMR_REMOTE_RD_MASK = Bit(28), 1388 OCRDMA_REG_NSMR_REMOTE_RD_MASK = BIT(28),
1391 OCRDMA_REG_NSMR_LOCAL_WR_SHIFT = 29, 1389 OCRDMA_REG_NSMR_LOCAL_WR_SHIFT = 29,
1392 OCRDMA_REG_NSMR_LOCAL_WR_MASK = Bit(29), 1390 OCRDMA_REG_NSMR_LOCAL_WR_MASK = BIT(29),
1393 OCRDMA_REG_NSMR_REMOTE_ATOMIC_SHIFT = 30, 1391 OCRDMA_REG_NSMR_REMOTE_ATOMIC_SHIFT = 30,
1394 OCRDMA_REG_NSMR_REMOTE_ATOMIC_MASK = Bit(30), 1392 OCRDMA_REG_NSMR_REMOTE_ATOMIC_MASK = BIT(30),
1395 OCRDMA_REG_NSMR_LAST_SHIFT = 31, 1393 OCRDMA_REG_NSMR_LAST_SHIFT = 31,
1396 OCRDMA_REG_NSMR_LAST_MASK = Bit(31) 1394 OCRDMA_REG_NSMR_LAST_MASK = BIT(31)
1397}; 1395};
1398 1396
1399struct ocrdma_reg_nsmr { 1397struct ocrdma_reg_nsmr {
@@ -1420,7 +1418,7 @@ enum {
1420 OCRDMA_REG_NSMR_CONT_NUM_PBL_SHIFT, 1418 OCRDMA_REG_NSMR_CONT_NUM_PBL_SHIFT,
1421 1419
1422 OCRDMA_REG_NSMR_CONT_LAST_SHIFT = 31, 1420 OCRDMA_REG_NSMR_CONT_LAST_SHIFT = 31,
1423 OCRDMA_REG_NSMR_CONT_LAST_MASK = Bit(31) 1421 OCRDMA_REG_NSMR_CONT_LAST_MASK = BIT(31)
1424}; 1422};
1425 1423
1426struct ocrdma_reg_nsmr_cont { 1424struct ocrdma_reg_nsmr_cont {
@@ -1566,7 +1564,7 @@ struct ocrdma_delete_ah_tbl_rsp {
1566 1564
1567enum { 1565enum {
1568 OCRDMA_EQE_VALID_SHIFT = 0, 1566 OCRDMA_EQE_VALID_SHIFT = 0,
1569 OCRDMA_EQE_VALID_MASK = Bit(0), 1567 OCRDMA_EQE_VALID_MASK = BIT(0),
1570 OCRDMA_EQE_FOR_CQE_MASK = 0xFFFE, 1568 OCRDMA_EQE_FOR_CQE_MASK = 0xFFFE,
1571 OCRDMA_EQE_RESOURCE_ID_SHIFT = 16, 1569 OCRDMA_EQE_RESOURCE_ID_SHIFT = 16,
1572 OCRDMA_EQE_RESOURCE_ID_MASK = 0xFFFF << 1570 OCRDMA_EQE_RESOURCE_ID_MASK = 0xFFFF <<
@@ -1624,11 +1622,11 @@ enum {
1624 OCRDMA_CQE_UD_STATUS_MASK = 0x7 << OCRDMA_CQE_UD_STATUS_SHIFT, 1622 OCRDMA_CQE_UD_STATUS_MASK = 0x7 << OCRDMA_CQE_UD_STATUS_SHIFT,
1625 OCRDMA_CQE_STATUS_SHIFT = 16, 1623 OCRDMA_CQE_STATUS_SHIFT = 16,
1626 OCRDMA_CQE_STATUS_MASK = 0xFF << OCRDMA_CQE_STATUS_SHIFT, 1624 OCRDMA_CQE_STATUS_MASK = 0xFF << OCRDMA_CQE_STATUS_SHIFT,
1627 OCRDMA_CQE_VALID = Bit(31), 1625 OCRDMA_CQE_VALID = BIT(31),
1628 OCRDMA_CQE_INVALIDATE = Bit(30), 1626 OCRDMA_CQE_INVALIDATE = BIT(30),
1629 OCRDMA_CQE_QTYPE = Bit(29), 1627 OCRDMA_CQE_QTYPE = BIT(29),
1630 OCRDMA_CQE_IMM = Bit(28), 1628 OCRDMA_CQE_IMM = BIT(28),
1631 OCRDMA_CQE_WRITE_IMM = Bit(27), 1629 OCRDMA_CQE_WRITE_IMM = BIT(27),
1632 OCRDMA_CQE_QTYPE_SQ = 0, 1630 OCRDMA_CQE_QTYPE_SQ = 0,
1633 OCRDMA_CQE_QTYPE_RQ = 1, 1631 OCRDMA_CQE_QTYPE_RQ = 1,
1634 OCRDMA_CQE_SRCQP_MASK = 0xFFFFFF 1632 OCRDMA_CQE_SRCQP_MASK = 0xFFFFFF
@@ -1772,8 +1770,8 @@ struct ocrdma_grh {
1772 u16 rsvd; 1770 u16 rsvd;
1773} __packed; 1771} __packed;
1774 1772
1775#define OCRDMA_AV_VALID Bit(7) 1773#define OCRDMA_AV_VALID BIT(7)
1776#define OCRDMA_AV_VLAN_VALID Bit(1) 1774#define OCRDMA_AV_VLAN_VALID BIT(1)
1777 1775
1778struct ocrdma_av { 1776struct ocrdma_av {
1779 struct ocrdma_eth_vlan eth_hdr; 1777 struct ocrdma_eth_vlan eth_hdr;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index e8b8569788c0..4c68305ee781 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -388,7 +388,7 @@ struct ib_ucontext *ocrdma_alloc_ucontext(struct ib_device *ibdev,
388 388
389 memset(&resp, 0, sizeof(resp)); 389 memset(&resp, 0, sizeof(resp));
390 resp.ah_tbl_len = ctx->ah_tbl.len; 390 resp.ah_tbl_len = ctx->ah_tbl.len;
391 resp.ah_tbl_page = ctx->ah_tbl.pa; 391 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va);
392 392
393 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len); 393 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);
394 if (status) 394 if (status)
@@ -870,7 +870,7 @@ static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
870 uresp.page_size = PAGE_ALIGN(cq->len); 870 uresp.page_size = PAGE_ALIGN(cq->len);
871 uresp.num_pages = 1; 871 uresp.num_pages = 1;
872 uresp.max_hw_cqe = cq->max_hw_cqe; 872 uresp.max_hw_cqe = cq->max_hw_cqe;
873 uresp.page_addr[0] = cq->pa; 873 uresp.page_addr[0] = virt_to_phys(cq->va);
874 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id); 874 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);
875 uresp.db_page_size = dev->nic_info.db_page_size; 875 uresp.db_page_size = dev->nic_info.db_page_size;
876 uresp.phase_change = cq->phase_change ? 1 : 0; 876 uresp.phase_change = cq->phase_change ? 1 : 0;
@@ -1123,13 +1123,13 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
1123 uresp.sq_dbid = qp->sq.dbid; 1123 uresp.sq_dbid = qp->sq.dbid;
1124 uresp.num_sq_pages = 1; 1124 uresp.num_sq_pages = 1;
1125 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len); 1125 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
1126 uresp.sq_page_addr[0] = qp->sq.pa; 1126 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va);
1127 uresp.num_wqe_allocated = qp->sq.max_cnt; 1127 uresp.num_wqe_allocated = qp->sq.max_cnt;
1128 if (!srq) { 1128 if (!srq) {
1129 uresp.rq_dbid = qp->rq.dbid; 1129 uresp.rq_dbid = qp->rq.dbid;
1130 uresp.num_rq_pages = 1; 1130 uresp.num_rq_pages = 1;
1131 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len); 1131 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
1132 uresp.rq_page_addr[0] = qp->rq.pa; 1132 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va);
1133 uresp.num_rqe_allocated = qp->rq.max_cnt; 1133 uresp.num_rqe_allocated = qp->rq.max_cnt;
1134 } 1134 }
1135 uresp.db_page_addr = usr_db; 1135 uresp.db_page_addr = usr_db;
@@ -1680,7 +1680,7 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
1680 memset(&uresp, 0, sizeof(uresp)); 1680 memset(&uresp, 0, sizeof(uresp));
1681 uresp.rq_dbid = srq->rq.dbid; 1681 uresp.rq_dbid = srq->rq.dbid;
1682 uresp.num_rq_pages = 1; 1682 uresp.num_rq_pages = 1;
1683 uresp.rq_page_addr[0] = srq->rq.pa; 1683 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va);
1684 uresp.rq_page_size = srq->rq.len; 1684 uresp.rq_page_size = srq->rq.len;
1685 uresp.db_page_addr = dev->nic_info.unmapped_db + 1685 uresp.db_page_addr = dev->nic_info.unmapped_db +
1686 (srq->pd->id * dev->nic_info.db_page_size); 1686 (srq->pd->id * dev->nic_info.db_page_size);
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 93ce62fe1594..f42ab14105ac 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -83,7 +83,7 @@ module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
83 83
84int iser_debug_level = 0; 84int iser_debug_level = 0;
85bool iser_pi_enable = false; 85bool iser_pi_enable = false;
86int iser_pi_guard = 0; 86int iser_pi_guard = 1;
87 87
88MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover"); 88MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover");
89MODULE_LICENSE("Dual BSD/GPL"); 89MODULE_LICENSE("Dual BSD/GPL");
@@ -97,14 +97,24 @@ module_param_named(pi_enable, iser_pi_enable, bool, 0644);
97MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)"); 97MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)");
98 98
99module_param_named(pi_guard, iser_pi_guard, int, 0644); 99module_param_named(pi_guard, iser_pi_guard, int, 0644);
100MODULE_PARM_DESC(pi_guard, "T10-PI guard_type, 0:CRC|1:IP_CSUM (default:CRC)"); 100MODULE_PARM_DESC(pi_guard, "T10-PI guard_type, 0:CRC|1:IP_CSUM (default:IP_CSUM)");
101 101
102static struct workqueue_struct *release_wq; 102static struct workqueue_struct *release_wq;
103struct iser_global ig; 103struct iser_global ig;
104 104
105/*
106 * iscsi_iser_recv() - Process a successfull recv completion
107 * @conn: iscsi connection
108 * @hdr: iscsi header
109 * @rx_data: buffer containing receive data payload
110 * @rx_data_len: length of rx_data
111 *
112 * Notes: In case of data length errors or iscsi PDU completion failures
113 * this routine will signal iscsi layer of connection failure.
114 */
105void 115void
106iscsi_iser_recv(struct iscsi_conn *conn, 116iscsi_iser_recv(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
107 struct iscsi_hdr *hdr, char *rx_data, int rx_data_len) 117 char *rx_data, int rx_data_len)
108{ 118{
109 int rc = 0; 119 int rc = 0;
110 int datalen; 120 int datalen;
@@ -135,20 +145,30 @@ error:
135 iscsi_conn_failure(conn, rc); 145 iscsi_conn_failure(conn, rc);
136} 146}
137 147
138static int iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode) 148/**
149 * iscsi_iser_pdu_alloc() - allocate an iscsi-iser PDU
150 * @task: iscsi task
151 * @opcode: iscsi command opcode
152 *
153 * Netes: This routine can't fail, just assign iscsi task
154 * hdr and max hdr size.
155 */
156static int
157iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
139{ 158{
140 struct iscsi_iser_task *iser_task = task->dd_data; 159 struct iscsi_iser_task *iser_task = task->dd_data;
141 160
142 task->hdr = (struct iscsi_hdr *)&iser_task->desc.iscsi_header; 161 task->hdr = (struct iscsi_hdr *)&iser_task->desc.iscsi_header;
143 task->hdr_max = sizeof(iser_task->desc.iscsi_header); 162 task->hdr_max = sizeof(iser_task->desc.iscsi_header);
163
144 return 0; 164 return 0;
145} 165}
146 166
147int iser_initialize_task_headers(struct iscsi_task *task, 167int iser_initialize_task_headers(struct iscsi_task *task,
148 struct iser_tx_desc *tx_desc) 168 struct iser_tx_desc *tx_desc)
149{ 169{
150 struct iser_conn *ib_conn = task->conn->dd_data; 170 struct iser_conn *iser_conn = task->conn->dd_data;
151 struct iser_device *device = ib_conn->device; 171 struct iser_device *device = iser_conn->ib_conn.device;
152 struct iscsi_iser_task *iser_task = task->dd_data; 172 struct iscsi_iser_task *iser_task = task->dd_data;
153 u64 dma_addr; 173 u64 dma_addr;
154 174
@@ -162,14 +182,18 @@ int iser_initialize_task_headers(struct iscsi_task *task,
162 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 182 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
163 tx_desc->tx_sg[0].lkey = device->mr->lkey; 183 tx_desc->tx_sg[0].lkey = device->mr->lkey;
164 184
165 iser_task->ib_conn = ib_conn; 185 iser_task->iser_conn = iser_conn;
166 return 0; 186 return 0;
167} 187}
188
168/** 189/**
169 * iscsi_iser_task_init - Initialize task 190 * iscsi_iser_task_init() - Initialize iscsi-iser task
170 * @task: iscsi task 191 * @task: iscsi task
171 * 192 *
172 * Initialize the task for the scsi command or mgmt command. 193 * Initialize the task for the scsi command or mgmt command.
194 *
195 * Return: Returns zero on success or -ENOMEM when failing
196 * to init task headers (dma mapping error).
173 */ 197 */
174static int 198static int
175iscsi_iser_task_init(struct iscsi_task *task) 199iscsi_iser_task_init(struct iscsi_task *task)
@@ -191,7 +215,7 @@ iscsi_iser_task_init(struct iscsi_task *task)
191} 215}
192 216
193/** 217/**
194 * iscsi_iser_mtask_xmit - xmit management(immediate) task 218 * iscsi_iser_mtask_xmit() - xmit management (immediate) task
195 * @conn: iscsi connection 219 * @conn: iscsi connection
196 * @task: task management task 220 * @task: task management task
197 * 221 *
@@ -249,6 +273,12 @@ iscsi_iser_task_xmit_unsol_data_exit:
249 return error; 273 return error;
250} 274}
251 275
276/**
277 * iscsi_iser_task_xmit() - xmit iscsi-iser task
278 * @task: iscsi task
279 *
280 * Return: zero on success or escalates $error on failure.
281 */
252static int 282static int
253iscsi_iser_task_xmit(struct iscsi_task *task) 283iscsi_iser_task_xmit(struct iscsi_task *task)
254{ 284{
@@ -286,12 +316,24 @@ iscsi_iser_task_xmit(struct iscsi_task *task)
286 return error; 316 return error;
287} 317}
288 318
319/**
320 * iscsi_iser_cleanup_task() - cleanup an iscsi-iser task
321 * @task: iscsi task
322 *
323 * Notes: In case the RDMA device is already NULL (might have
324 * been removed in DEVICE_REMOVAL CM event it will bail-out
325 * without doing dma unmapping.
326 */
289static void iscsi_iser_cleanup_task(struct iscsi_task *task) 327static void iscsi_iser_cleanup_task(struct iscsi_task *task)
290{ 328{
291 struct iscsi_iser_task *iser_task = task->dd_data; 329 struct iscsi_iser_task *iser_task = task->dd_data;
292 struct iser_tx_desc *tx_desc = &iser_task->desc; 330 struct iser_tx_desc *tx_desc = &iser_task->desc;
293 struct iser_conn *ib_conn = task->conn->dd_data; 331 struct iser_conn *iser_conn = task->conn->dd_data;
294 struct iser_device *device = ib_conn->device; 332 struct iser_device *device = iser_conn->ib_conn.device;
333
334 /* DEVICE_REMOVAL event might have already released the device */
335 if (!device)
336 return;
295 337
296 ib_dma_unmap_single(device->ib_device, 338 ib_dma_unmap_single(device->ib_device,
297 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 339 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
@@ -306,7 +348,20 @@ static void iscsi_iser_cleanup_task(struct iscsi_task *task)
306 } 348 }
307} 349}
308 350
309static u8 iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector) 351/**
352 * iscsi_iser_check_protection() - check protection information status of task.
353 * @task: iscsi task
354 * @sector: error sector if exsists (output)
355 *
356 * Return: zero if no data-integrity errors have occured
357 * 0x1: data-integrity error occured in the guard-block
358 * 0x2: data-integrity error occured in the reference tag
359 * 0x3: data-integrity error occured in the application tag
360 *
361 * In addition the error sector is marked.
362 */
363static u8
364iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector)
310{ 365{
311 struct iscsi_iser_task *iser_task = task->dd_data; 366 struct iscsi_iser_task *iser_task = task->dd_data;
312 367
@@ -318,8 +373,17 @@ static u8 iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector)
318 sector); 373 sector);
319} 374}
320 375
376/**
377 * iscsi_iser_conn_create() - create a new iscsi-iser connection
378 * @cls_session: iscsi class connection
379 * @conn_idx: connection index within the session (for MCS)
380 *
381 * Return: iscsi_cls_conn when iscsi_conn_setup succeeds or NULL
382 * otherwise.
383 */
321static struct iscsi_cls_conn * 384static struct iscsi_cls_conn *
322iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx) 385iscsi_iser_conn_create(struct iscsi_cls_session *cls_session,
386 uint32_t conn_idx)
323{ 387{
324 struct iscsi_conn *conn; 388 struct iscsi_conn *conn;
325 struct iscsi_cls_conn *cls_conn; 389 struct iscsi_cls_conn *cls_conn;
@@ -338,13 +402,25 @@ iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
338 return cls_conn; 402 return cls_conn;
339} 403}
340 404
405/**
406 * iscsi_iser_conn_bind() - bind iscsi and iser connection structures
407 * @cls_session: iscsi class session
408 * @cls_conn: iscsi class connection
409 * @transport_eph: transport end-point handle
410 * @is_leading: indicate if this is the session leading connection (MCS)
411 *
412 * Return: zero on success, $error if iscsi_conn_bind fails and
413 * -EINVAL in case end-point doesn't exsits anymore or iser connection
414 * state is not UP (teardown already started).
415 */
341static int 416static int
342iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session, 417iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
343 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 418 struct iscsi_cls_conn *cls_conn,
419 uint64_t transport_eph,
344 int is_leading) 420 int is_leading)
345{ 421{
346 struct iscsi_conn *conn = cls_conn->dd_data; 422 struct iscsi_conn *conn = cls_conn->dd_data;
347 struct iser_conn *ib_conn; 423 struct iser_conn *iser_conn;
348 struct iscsi_endpoint *ep; 424 struct iscsi_endpoint *ep;
349 int error; 425 int error;
350 426
@@ -360,66 +436,100 @@ iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
360 (unsigned long long)transport_eph); 436 (unsigned long long)transport_eph);
361 return -EINVAL; 437 return -EINVAL;
362 } 438 }
363 ib_conn = ep->dd_data; 439 iser_conn = ep->dd_data;
364 440
365 mutex_lock(&ib_conn->state_mutex); 441 mutex_lock(&iser_conn->state_mutex);
366 if (ib_conn->state != ISER_CONN_UP) { 442 if (iser_conn->state != ISER_CONN_UP) {
367 error = -EINVAL; 443 error = -EINVAL;
368 iser_err("iser_conn %p state is %d, teardown started\n", 444 iser_err("iser_conn %p state is %d, teardown started\n",
369 ib_conn, ib_conn->state); 445 iser_conn, iser_conn->state);
370 goto out; 446 goto out;
371 } 447 }
372 448
373 error = iser_alloc_rx_descriptors(ib_conn, conn->session); 449 error = iser_alloc_rx_descriptors(iser_conn, conn->session);
374 if (error) 450 if (error)
375 goto out; 451 goto out;
376 452
377 /* binds the iSER connection retrieved from the previously 453 /* binds the iSER connection retrieved from the previously
378 * connected ep_handle to the iSCSI layer connection. exchanges 454 * connected ep_handle to the iSCSI layer connection. exchanges
379 * connection pointers */ 455 * connection pointers */
380 iser_info("binding iscsi conn %p to ib_conn %p\n", conn, ib_conn); 456 iser_info("binding iscsi conn %p to iser_conn %p\n", conn, iser_conn);
381 457
382 conn->dd_data = ib_conn; 458 conn->dd_data = iser_conn;
383 ib_conn->iscsi_conn = conn; 459 iser_conn->iscsi_conn = conn;
384 460
385out: 461out:
386 mutex_unlock(&ib_conn->state_mutex); 462 mutex_unlock(&iser_conn->state_mutex);
387 return error; 463 return error;
388} 464}
389 465
466/**
467 * iscsi_iser_conn_start() - start iscsi-iser connection
468 * @cls_conn: iscsi class connection
469 *
470 * Notes: Here iser intialize (or re-initialize) stop_completion as
471 * from this point iscsi must call conn_stop in session/connection
472 * teardown so iser transport must wait for it.
473 */
390static int 474static int
391iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn) 475iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn)
392{ 476{
393 struct iscsi_conn *iscsi_conn; 477 struct iscsi_conn *iscsi_conn;
394 struct iser_conn *ib_conn; 478 struct iser_conn *iser_conn;
395 479
396 iscsi_conn = cls_conn->dd_data; 480 iscsi_conn = cls_conn->dd_data;
397 ib_conn = iscsi_conn->dd_data; 481 iser_conn = iscsi_conn->dd_data;
398 reinit_completion(&ib_conn->stop_completion); 482 reinit_completion(&iser_conn->stop_completion);
399 483
400 return iscsi_conn_start(cls_conn); 484 return iscsi_conn_start(cls_conn);
401} 485}
402 486
487/**
488 * iscsi_iser_conn_stop() - stop iscsi-iser connection
489 * @cls_conn: iscsi class connection
490 * @flag: indicate if recover or terminate (passed as is)
491 *
492 * Notes: Calling iscsi_conn_stop might theoretically race with
493 * DEVICE_REMOVAL event and dereference a previously freed RDMA device
494 * handle, so we call it under iser the state lock to protect against
495 * this kind of race.
496 */
403static void 497static void
404iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 498iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
405{ 499{
406 struct iscsi_conn *conn = cls_conn->dd_data; 500 struct iscsi_conn *conn = cls_conn->dd_data;
407 struct iser_conn *ib_conn = conn->dd_data; 501 struct iser_conn *iser_conn = conn->dd_data;
408 502
409 iser_dbg("stopping iscsi_conn: %p, ib_conn: %p\n", conn, ib_conn); 503 iser_info("stopping iscsi_conn: %p, iser_conn: %p\n", conn, iser_conn);
410 iscsi_conn_stop(cls_conn, flag);
411 504
412 /* 505 /*
413 * Userspace may have goofed up and not bound the connection or 506 * Userspace may have goofed up and not bound the connection or
414 * might have only partially setup the connection. 507 * might have only partially setup the connection.
415 */ 508 */
416 if (ib_conn) { 509 if (iser_conn) {
510 mutex_lock(&iser_conn->state_mutex);
511 iscsi_conn_stop(cls_conn, flag);
512 iser_conn_terminate(iser_conn);
513
514 /* unbind */
515 iser_conn->iscsi_conn = NULL;
417 conn->dd_data = NULL; 516 conn->dd_data = NULL;
418 complete(&ib_conn->stop_completion); 517
518 complete(&iser_conn->stop_completion);
519 mutex_unlock(&iser_conn->state_mutex);
520 } else {
521 iscsi_conn_stop(cls_conn, flag);
419 } 522 }
420} 523}
421 524
422static void iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session) 525/**
526 * iscsi_iser_session_destroy() - destroy iscsi-iser session
527 * @cls_session: iscsi class session
528 *
529 * Removes and free iscsi host.
530 */
531static void
532iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session)
423{ 533{
424 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 534 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
425 535
@@ -439,6 +549,16 @@ iser_dif_prot_caps(int prot_caps)
439 SHOST_DIX_TYPE3_PROTECTION : 0); 549 SHOST_DIX_TYPE3_PROTECTION : 0);
440} 550}
441 551
552/**
553 * iscsi_iser_session_create() - create an iscsi-iser session
554 * @ep: iscsi end-point handle
555 * @cmds_max: maximum commands in this session
556 * @qdepth: session command queue depth
557 * @initial_cmdsn: initiator command sequnce number
558 *
559 * Allocates and adds a scsi host, expose DIF supprot if
560 * exists, and sets up an iscsi session.
561 */
442static struct iscsi_cls_session * 562static struct iscsi_cls_session *
443iscsi_iser_session_create(struct iscsi_endpoint *ep, 563iscsi_iser_session_create(struct iscsi_endpoint *ep,
444 uint16_t cmds_max, uint16_t qdepth, 564 uint16_t cmds_max, uint16_t qdepth,
@@ -447,7 +567,8 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
447 struct iscsi_cls_session *cls_session; 567 struct iscsi_cls_session *cls_session;
448 struct iscsi_session *session; 568 struct iscsi_session *session;
449 struct Scsi_Host *shost; 569 struct Scsi_Host *shost;
450 struct iser_conn *ib_conn = NULL; 570 struct iser_conn *iser_conn = NULL;
571 struct ib_conn *ib_conn;
451 572
452 shost = iscsi_host_alloc(&iscsi_iser_sht, 0, 0); 573 shost = iscsi_host_alloc(&iscsi_iser_sht, 0, 0);
453 if (!shost) 574 if (!shost)
@@ -464,7 +585,8 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
464 * the leading conn's ep so this will be NULL; 585 * the leading conn's ep so this will be NULL;
465 */ 586 */
466 if (ep) { 587 if (ep) {
467 ib_conn = ep->dd_data; 588 iser_conn = ep->dd_data;
589 ib_conn = &iser_conn->ib_conn;
468 if (ib_conn->pi_support) { 590 if (ib_conn->pi_support) {
469 u32 sig_caps = ib_conn->device->dev_attr.sig_prot_cap; 591 u32 sig_caps = ib_conn->device->dev_attr.sig_prot_cap;
470 592
@@ -476,8 +598,8 @@ iscsi_iser_session_create(struct iscsi_endpoint *ep,
476 } 598 }
477 } 599 }
478 600
479 if (iscsi_host_add(shost, 601 if (iscsi_host_add(shost, ep ?
480 ep ? ib_conn->device->ib_device->dma_device : NULL)) 602 ib_conn->device->ib_device->dma_device : NULL))
481 goto free_host; 603 goto free_host;
482 604
483 if (cmds_max > ISER_DEF_XMIT_CMDS_MAX) { 605 if (cmds_max > ISER_DEF_XMIT_CMDS_MAX) {
@@ -549,6 +671,13 @@ iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn,
549 return 0; 671 return 0;
550} 672}
551 673
674/**
675 * iscsi_iser_set_param() - set class connection parameter
676 * @cls_conn: iscsi class connection
677 * @stats: iscsi stats to output
678 *
679 * Output connection statistics.
680 */
552static void 681static void
553iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats) 682iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
554{ 683{
@@ -577,18 +706,18 @@ iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *s
577static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep, 706static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep,
578 enum iscsi_param param, char *buf) 707 enum iscsi_param param, char *buf)
579{ 708{
580 struct iser_conn *ib_conn = ep->dd_data; 709 struct iser_conn *iser_conn = ep->dd_data;
581 int len; 710 int len;
582 711
583 switch (param) { 712 switch (param) {
584 case ISCSI_PARAM_CONN_PORT: 713 case ISCSI_PARAM_CONN_PORT:
585 case ISCSI_PARAM_CONN_ADDRESS: 714 case ISCSI_PARAM_CONN_ADDRESS:
586 if (!ib_conn || !ib_conn->cma_id) 715 if (!iser_conn || !iser_conn->ib_conn.cma_id)
587 return -ENOTCONN; 716 return -ENOTCONN;
588 717
589 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 718 return iscsi_conn_get_addr_param((struct sockaddr_storage *)
590 &ib_conn->cma_id->route.addr.dst_addr, 719 &iser_conn->ib_conn.cma_id->route.addr.dst_addr,
591 param, buf); 720 param, buf);
592 break; 721 break;
593 default: 722 default:
594 return -ENOSYS; 723 return -ENOSYS;
@@ -597,29 +726,44 @@ static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep,
597 return len; 726 return len;
598} 727}
599 728
729/**
730 * iscsi_iser_ep_connect() - Initiate iSER connection establishment
731 * @shost: scsi_host
732 * @dst_addr: destination address
733 * @non-blocking: indicate if routine can block
734 *
735 * Allocate an iscsi endpoint, an iser_conn structure and bind them.
736 * After that start RDMA connection establishment via rdma_cm. We
737 * don't allocate iser_conn embedded in iscsi_endpoint since in teardown
738 * the endpoint will be destroyed at ep_disconnect while iser_conn will
739 * cleanup its resources asynchronuously.
740 *
741 * Return: iscsi_endpoint created by iscsi layer or ERR_PTR(error)
742 * if fails.
743 */
600static struct iscsi_endpoint * 744static struct iscsi_endpoint *
601iscsi_iser_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr, 745iscsi_iser_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
602 int non_blocking) 746 int non_blocking)
603{ 747{
604 int err; 748 int err;
605 struct iser_conn *ib_conn; 749 struct iser_conn *iser_conn;
606 struct iscsi_endpoint *ep; 750 struct iscsi_endpoint *ep;
607 751
608 ep = iscsi_create_endpoint(0); 752 ep = iscsi_create_endpoint(0);
609 if (!ep) 753 if (!ep)
610 return ERR_PTR(-ENOMEM); 754 return ERR_PTR(-ENOMEM);
611 755
612 ib_conn = kzalloc(sizeof(*ib_conn), GFP_KERNEL); 756 iser_conn = kzalloc(sizeof(*iser_conn), GFP_KERNEL);
613 if (!ib_conn) { 757 if (!iser_conn) {
614 err = -ENOMEM; 758 err = -ENOMEM;
615 goto failure; 759 goto failure;
616 } 760 }
617 761
618 ep->dd_data = ib_conn; 762 ep->dd_data = iser_conn;
619 ib_conn->ep = ep; 763 iser_conn->ep = ep;
620 iser_conn_init(ib_conn); 764 iser_conn_init(iser_conn);
621 765
622 err = iser_connect(ib_conn, NULL, dst_addr, non_blocking); 766 err = iser_connect(iser_conn, NULL, dst_addr, non_blocking);
623 if (err) 767 if (err)
624 goto failure; 768 goto failure;
625 769
@@ -629,25 +773,38 @@ failure:
629 return ERR_PTR(err); 773 return ERR_PTR(err);
630} 774}
631 775
776/**
777 * iscsi_iser_ep_poll() - poll for iser connection establishment to complete
778 * @ep: iscsi endpoint (created at ep_connect)
779 * @timeout_ms: polling timeout allowed in ms.
780 *
781 * This routine boils down to waiting for up_completion signaling
782 * that cma_id got CONNECTED event.
783 *
784 * Return: 1 if succeeded in connection establishment, 0 if timeout expired
785 * (libiscsi will retry will kick in) or -1 if interrupted by signal
786 * or more likely iser connection state transitioned to TEMINATING or
787 * DOWN during the wait period.
788 */
632static int 789static int
633iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) 790iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
634{ 791{
635 struct iser_conn *ib_conn; 792 struct iser_conn *iser_conn;
636 int rc; 793 int rc;
637 794
638 ib_conn = ep->dd_data; 795 iser_conn = ep->dd_data;
639 rc = wait_for_completion_interruptible_timeout(&ib_conn->up_completion, 796 rc = wait_for_completion_interruptible_timeout(&iser_conn->up_completion,
640 msecs_to_jiffies(timeout_ms)); 797 msecs_to_jiffies(timeout_ms));
641 /* if conn establishment failed, return error code to iscsi */ 798 /* if conn establishment failed, return error code to iscsi */
642 if (rc == 0) { 799 if (rc == 0) {
643 mutex_lock(&ib_conn->state_mutex); 800 mutex_lock(&iser_conn->state_mutex);
644 if (ib_conn->state == ISER_CONN_TERMINATING || 801 if (iser_conn->state == ISER_CONN_TERMINATING ||
645 ib_conn->state == ISER_CONN_DOWN) 802 iser_conn->state == ISER_CONN_DOWN)
646 rc = -1; 803 rc = -1;
647 mutex_unlock(&ib_conn->state_mutex); 804 mutex_unlock(&iser_conn->state_mutex);
648 } 805 }
649 806
650 iser_info("ib conn %p rc = %d\n", ib_conn, rc); 807 iser_info("ib conn %p rc = %d\n", iser_conn, rc);
651 808
652 if (rc > 0) 809 if (rc > 0)
653 return 1; /* success, this is the equivalent of POLLOUT */ 810 return 1; /* success, this is the equivalent of POLLOUT */
@@ -657,15 +814,26 @@ iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
657 return rc; /* signal */ 814 return rc; /* signal */
658} 815}
659 816
817/**
818 * iscsi_iser_ep_disconnect() - Initiate connection teardown process
819 * @ep: iscsi endpoint handle
820 *
821 * This routine is not blocked by iser and RDMA termination process
822 * completion as we queue a deffered work for iser/RDMA destruction
823 * and cleanup or actually call it immediately in case we didn't pass
824 * iscsi conn bind/start stage, thus it is safe.
825 */
660static void 826static void
661iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep) 827iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep)
662{ 828{
663 struct iser_conn *ib_conn; 829 struct iser_conn *iser_conn;
830
831 iser_conn = ep->dd_data;
832 iser_info("ep %p iser conn %p state %d\n",
833 ep, iser_conn, iser_conn->state);
664 834
665 ib_conn = ep->dd_data; 835 mutex_lock(&iser_conn->state_mutex);
666 iser_info("ep %p ib conn %p state %d\n", ep, ib_conn, ib_conn->state); 836 iser_conn_terminate(iser_conn);
667 mutex_lock(&ib_conn->state_mutex);
668 iser_conn_terminate(ib_conn);
669 837
670 /* 838 /*
671 * if iser_conn and iscsi_conn are bound, we must wait for 839 * if iser_conn and iscsi_conn are bound, we must wait for
@@ -673,14 +841,14 @@ iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep)
673 * the iser resources. Otherwise we are safe to free resources 841 * the iser resources. Otherwise we are safe to free resources
674 * immediately. 842 * immediately.
675 */ 843 */
676 if (ib_conn->iscsi_conn) { 844 if (iser_conn->iscsi_conn) {
677 INIT_WORK(&ib_conn->release_work, iser_release_work); 845 INIT_WORK(&iser_conn->release_work, iser_release_work);
678 queue_work(release_wq, &ib_conn->release_work); 846 queue_work(release_wq, &iser_conn->release_work);
679 mutex_unlock(&ib_conn->state_mutex); 847 mutex_unlock(&iser_conn->state_mutex);
680 } else { 848 } else {
681 ib_conn->state = ISER_CONN_DOWN; 849 iser_conn->state = ISER_CONN_DOWN;
682 mutex_unlock(&ib_conn->state_mutex); 850 mutex_unlock(&iser_conn->state_mutex);
683 iser_conn_release(ib_conn); 851 iser_conn_release(iser_conn);
684 } 852 }
685 iscsi_destroy_endpoint(ep); 853 iscsi_destroy_endpoint(ep);
686} 854}
@@ -843,7 +1011,7 @@ register_transport_failure:
843 1011
844static void __exit iser_exit(void) 1012static void __exit iser_exit(void)
845{ 1013{
846 struct iser_conn *ib_conn, *n; 1014 struct iser_conn *iser_conn, *n;
847 int connlist_empty; 1015 int connlist_empty;
848 1016
849 iser_dbg("Removing iSER datamover...\n"); 1017 iser_dbg("Removing iSER datamover...\n");
@@ -856,8 +1024,9 @@ static void __exit iser_exit(void)
856 if (!connlist_empty) { 1024 if (!connlist_empty) {
857 iser_err("Error cleanup stage completed but we still have iser " 1025 iser_err("Error cleanup stage completed but we still have iser "
858 "connections, destroying them anyway.\n"); 1026 "connections, destroying them anyway.\n");
859 list_for_each_entry_safe(ib_conn, n, &ig.connlist, conn_list) { 1027 list_for_each_entry_safe(iser_conn, n, &ig.connlist,
860 iser_conn_release(ib_conn); 1028 conn_list) {
1029 iser_conn_release(iser_conn);
861 } 1030 }
862 } 1031 }
863 1032
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 9f0e0e34d6ca..cd4174ca9a76 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -69,39 +69,38 @@
69 69
70#define DRV_NAME "iser" 70#define DRV_NAME "iser"
71#define PFX DRV_NAME ": " 71#define PFX DRV_NAME ": "
72#define DRV_VER "1.4.1" 72#define DRV_VER "1.4.8"
73 73
74#define iser_dbg(fmt, arg...) \ 74#define iser_dbg(fmt, arg...) \
75 do { \ 75 do { \
76 if (iser_debug_level > 2) \ 76 if (iser_debug_level > 2) \
77 printk(KERN_DEBUG PFX "%s:" fmt,\ 77 printk(KERN_DEBUG PFX "%s: " fmt,\
78 __func__ , ## arg); \ 78 __func__ , ## arg); \
79 } while (0) 79 } while (0)
80 80
81#define iser_warn(fmt, arg...) \ 81#define iser_warn(fmt, arg...) \
82 do { \ 82 do { \
83 if (iser_debug_level > 0) \ 83 if (iser_debug_level > 0) \
84 pr_warn(PFX "%s:" fmt, \ 84 pr_warn(PFX "%s: " fmt, \
85 __func__ , ## arg); \ 85 __func__ , ## arg); \
86 } while (0) 86 } while (0)
87 87
88#define iser_info(fmt, arg...) \ 88#define iser_info(fmt, arg...) \
89 do { \ 89 do { \
90 if (iser_debug_level > 1) \ 90 if (iser_debug_level > 1) \
91 pr_info(PFX "%s:" fmt, \ 91 pr_info(PFX "%s: " fmt, \
92 __func__ , ## arg); \ 92 __func__ , ## arg); \
93 } while (0) 93 } while (0)
94 94
95#define iser_err(fmt, arg...) \ 95#define iser_err(fmt, arg...) \
96 do { \ 96 do { \
97 printk(KERN_ERR PFX "%s:" fmt, \ 97 printk(KERN_ERR PFX "%s: " fmt, \
98 __func__ , ## arg); \ 98 __func__ , ## arg); \
99 } while (0) 99 } while (0)
100 100
101#define SHIFT_4K 12 101#define SHIFT_4K 12
102#define SIZE_4K (1ULL << SHIFT_4K) 102#define SIZE_4K (1ULL << SHIFT_4K)
103#define MASK_4K (~(SIZE_4K-1)) 103#define MASK_4K (~(SIZE_4K-1))
104
105 /* support up to 512KB in one RDMA */ 104 /* support up to 512KB in one RDMA */
106#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K) 105#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K)
107#define ISER_DEF_XMIT_CMDS_DEFAULT 512 106#define ISER_DEF_XMIT_CMDS_DEFAULT 512
@@ -145,18 +144,32 @@
145 ISER_MAX_TX_MISC_PDUS + \ 144 ISER_MAX_TX_MISC_PDUS + \
146 ISER_MAX_RX_MISC_PDUS) 145 ISER_MAX_RX_MISC_PDUS)
147 146
147#define ISER_WC_BATCH_COUNT 16
148#define ISER_SIGNAL_CMD_COUNT 32
149
148#define ISER_VER 0x10 150#define ISER_VER 0x10
149#define ISER_WSV 0x08 151#define ISER_WSV 0x08
150#define ISER_RSV 0x04 152#define ISER_RSV 0x04
151 153
152#define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL 154#define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL
155#define ISER_BEACON_WRID 0xfffffffffffffffeULL
153 156
157/**
158 * struct iser_hdr - iSER header
159 *
160 * @flags: flags support (zbva, remote_inv)
161 * @rsvd: reserved
162 * @write_stag: write rkey
163 * @write_va: write virtual address
164 * @reaf_stag: read rkey
165 * @read_va: read virtual address
166 */
154struct iser_hdr { 167struct iser_hdr {
155 u8 flags; 168 u8 flags;
156 u8 rsvd[3]; 169 u8 rsvd[3];
157 __be32 write_stag; /* write rkey */ 170 __be32 write_stag;
158 __be64 write_va; 171 __be64 write_va;
159 __be32 read_stag; /* read rkey */ 172 __be32 read_stag;
160 __be64 read_va; 173 __be64 read_va;
161} __attribute__((packed)); 174} __attribute__((packed));
162 175
@@ -179,7 +192,7 @@ struct iser_cm_hdr {
179/* Length of an object name string */ 192/* Length of an object name string */
180#define ISER_OBJECT_NAME_SIZE 64 193#define ISER_OBJECT_NAME_SIZE 64
181 194
182enum iser_ib_conn_state { 195enum iser_conn_state {
183 ISER_CONN_INIT, /* descriptor allocd, no conn */ 196 ISER_CONN_INIT, /* descriptor allocd, no conn */
184 ISER_CONN_PENDING, /* in the process of being established */ 197 ISER_CONN_PENDING, /* in the process of being established */
185 ISER_CONN_UP, /* up and running */ 198 ISER_CONN_UP, /* up and running */
@@ -200,23 +213,42 @@ enum iser_data_dir {
200 ISER_DIRS_NUM 213 ISER_DIRS_NUM
201}; 214};
202 215
216/**
217 * struct iser_data_buf - iSER data buffer
218 *
219 * @buf: pointer to the sg list
220 * @size: num entries of this sg
221 * @data_len: total beffer byte len
222 * @dma_nents: returned by dma_map_sg
223 * @copy_buf: allocated copy buf for SGs unaligned
224 * for rdma which are copied
225 * @sg_single: SG-ified clone of a non SG SC or
226 * unaligned SG
227 */
203struct iser_data_buf { 228struct iser_data_buf {
204 void *buf; /* pointer to the sg list */ 229 void *buf;
205 unsigned int size; /* num entries of this sg */ 230 unsigned int size;
206 unsigned long data_len; /* total data len */ 231 unsigned long data_len;
207 unsigned int dma_nents; /* returned by dma_map_sg */ 232 unsigned int dma_nents;
208 char *copy_buf; /* allocated copy buf for SGs unaligned * 233 char *copy_buf;
209 * for rdma which are copied */ 234 struct scatterlist sg_single;
210 struct scatterlist sg_single; /* SG-ified clone of a non SG SC or *
211 * unaligned SG */
212 }; 235 };
213 236
214/* fwd declarations */ 237/* fwd declarations */
215struct iser_device; 238struct iser_device;
216struct iser_cq_desc;
217struct iscsi_iser_task; 239struct iscsi_iser_task;
218struct iscsi_endpoint; 240struct iscsi_endpoint;
219 241
242/**
243 * struct iser_mem_reg - iSER memory registration info
244 *
245 * @lkey: MR local key
246 * @rkey: MR remote key
247 * @va: MR start address (buffer va)
248 * @len: MR length
249 * @mem_h: pointer to registration context (FMR/Fastreg)
250 * @is_mr: indicates weather we registered the buffer
251 */
220struct iser_mem_reg { 252struct iser_mem_reg {
221 u32 lkey; 253 u32 lkey;
222 u32 rkey; 254 u32 rkey;
@@ -226,11 +258,20 @@ struct iser_mem_reg {
226 int is_mr; 258 int is_mr;
227}; 259};
228 260
261/**
262 * struct iser_regd_buf - iSER buffer registration desc
263 *
264 * @reg: memory registration info
265 * @virt_addr: virtual address of buffer
266 * @device: reference to iser device
267 * @direction: dma direction (for dma_unmap)
268 * @data_size: data buffer size in bytes
269 */
229struct iser_regd_buf { 270struct iser_regd_buf {
230 struct iser_mem_reg reg; /* memory registration info */ 271 struct iser_mem_reg reg;
231 void *virt_addr; 272 void *virt_addr;
232 struct iser_device *device; /* device->device for dma_unmap */ 273 struct iser_device *device;
233 enum dma_data_direction direction; /* direction for dma_unmap */ 274 enum dma_data_direction direction;
234 unsigned int data_size; 275 unsigned int data_size;
235}; 276};
236 277
@@ -240,19 +281,39 @@ enum iser_desc_type {
240 ISCSI_TX_DATAOUT 281 ISCSI_TX_DATAOUT
241}; 282};
242 283
284/**
285 * struct iser_tx_desc - iSER TX descriptor (for send wr_id)
286 *
287 * @iser_header: iser header
288 * @iscsi_header: iscsi header
289 * @type: command/control/dataout
290 * @dam_addr: header buffer dma_address
291 * @tx_sg: sg[0] points to iser/iscsi headers
292 * sg[1] optionally points to either of immediate data
293 * unsolicited data-out or control
294 * @num_sge: number sges used on this TX task
295 */
243struct iser_tx_desc { 296struct iser_tx_desc {
244 struct iser_hdr iser_header; 297 struct iser_hdr iser_header;
245 struct iscsi_hdr iscsi_header; 298 struct iscsi_hdr iscsi_header;
246 enum iser_desc_type type; 299 enum iser_desc_type type;
247 u64 dma_addr; 300 u64 dma_addr;
248 /* sg[0] points to iser/iscsi headers, sg[1] optionally points to either
249 of immediate data, unsolicited data-out or control (login,text) */
250 struct ib_sge tx_sg[2]; 301 struct ib_sge tx_sg[2];
251 int num_sge; 302 int num_sge;
252}; 303};
253 304
254#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \ 305#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
255 sizeof(u64) + sizeof(struct ib_sge))) 306 sizeof(u64) + sizeof(struct ib_sge)))
307/**
308 * struct iser_rx_desc - iSER RX descriptor (for recv wr_id)
309 *
310 * @iser_header: iser header
311 * @iscsi_header: iscsi header
312 * @data: received data segment
313 * @dma_addr: receive buffer dma address
314 * @rx_sg: ib_sge of receive buffer
315 * @pad: for sense data TODO: Modify to maximum sense length supported
316 */
256struct iser_rx_desc { 317struct iser_rx_desc {
257 struct iser_hdr iser_header; 318 struct iser_hdr iser_header;
258 struct iscsi_hdr iscsi_header; 319 struct iscsi_hdr iscsi_header;
@@ -265,25 +326,59 @@ struct iser_rx_desc {
265#define ISER_MAX_CQ 4 326#define ISER_MAX_CQ 4
266 327
267struct iser_conn; 328struct iser_conn;
329struct ib_conn;
268struct iscsi_iser_task; 330struct iscsi_iser_task;
269 331
332/**
333 * struct iser_comp - iSER completion context
334 *
335 * @device: pointer to device handle
336 * @cq: completion queue
337 * @wcs: work completion array
338 * @tasklet: Tasklet handle
339 * @active_qps: Number of active QPs attached
340 * to completion context
341 */
342struct iser_comp {
343 struct iser_device *device;
344 struct ib_cq *cq;
345 struct ib_wc wcs[ISER_WC_BATCH_COUNT];
346 struct tasklet_struct tasklet;
347 int active_qps;
348};
349
350/**
351 * struct iser_device - iSER device handle
352 *
353 * @ib_device: RDMA device
354 * @pd: Protection Domain for this device
355 * @dev_attr: Device attributes container
356 * @mr: Global DMA memory region
357 * @event_handler: IB events handle routine
358 * @ig_list: entry in devices list
359 * @refcount: Reference counter, dominated by open iser connections
360 * @comps_used: Number of completion contexts used, Min between online
361 * cpus and device max completion vectors
362 * @comps: Dinamically allocated array of completion handlers
363 * Memory registration pool Function pointers (FMR or Fastreg):
364 * @iser_alloc_rdma_reg_res: Allocation of memory regions pool
365 * @iser_free_rdma_reg_res: Free of memory regions pool
366 * @iser_reg_rdma_mem: Memory registration routine
367 * @iser_unreg_rdma_mem: Memory deregistration routine
368 */
270struct iser_device { 369struct iser_device {
271 struct ib_device *ib_device; 370 struct ib_device *ib_device;
272 struct ib_pd *pd; 371 struct ib_pd *pd;
273 struct ib_device_attr dev_attr; 372 struct ib_device_attr dev_attr;
274 struct ib_cq *rx_cq[ISER_MAX_CQ];
275 struct ib_cq *tx_cq[ISER_MAX_CQ];
276 struct ib_mr *mr; 373 struct ib_mr *mr;
277 struct tasklet_struct cq_tasklet[ISER_MAX_CQ];
278 struct ib_event_handler event_handler; 374 struct ib_event_handler event_handler;
279 struct list_head ig_list; /* entry in ig devices list */ 375 struct list_head ig_list;
280 int refcount; 376 int refcount;
281 int cq_active_qps[ISER_MAX_CQ]; 377 int comps_used;
282 int cqs_used; 378 struct iser_comp comps[ISER_MAX_CQ];
283 struct iser_cq_desc *cq_desc; 379 int (*iser_alloc_rdma_reg_res)(struct ib_conn *ib_conn,
284 int (*iser_alloc_rdma_reg_res)(struct iser_conn *ib_conn,
285 unsigned cmds_max); 380 unsigned cmds_max);
286 void (*iser_free_rdma_reg_res)(struct iser_conn *ib_conn); 381 void (*iser_free_rdma_reg_res)(struct ib_conn *ib_conn);
287 int (*iser_reg_rdma_mem)(struct iscsi_iser_task *iser_task, 382 int (*iser_reg_rdma_mem)(struct iscsi_iser_task *iser_task,
288 enum iser_data_dir cmd_dir); 383 enum iser_data_dir cmd_dir);
289 void (*iser_unreg_rdma_mem)(struct iscsi_iser_task *iser_task, 384 void (*iser_unreg_rdma_mem)(struct iscsi_iser_task *iser_task,
@@ -301,78 +396,160 @@ enum iser_reg_indicator {
301 ISER_FASTREG_PROTECTED = 1 << 3, 396 ISER_FASTREG_PROTECTED = 1 << 3,
302}; 397};
303 398
399/**
400 * struct iser_pi_context - Protection information context
401 *
402 * @prot_mr: protection memory region
403 * @prot_frpl: protection fastreg page list
404 * @sig_mr: signature feature enabled memory region
405 */
304struct iser_pi_context { 406struct iser_pi_context {
305 struct ib_mr *prot_mr; 407 struct ib_mr *prot_mr;
306 struct ib_fast_reg_page_list *prot_frpl; 408 struct ib_fast_reg_page_list *prot_frpl;
307 struct ib_mr *sig_mr; 409 struct ib_mr *sig_mr;
308}; 410};
309 411
412/**
413 * struct fast_reg_descriptor - Fast registration descriptor
414 *
415 * @list: entry in connection fastreg pool
416 * @data_mr: data memory region
417 * @data_frpl: data fastreg page list
418 * @pi_ctx: protection information context
419 * @reg_indicators: fast registration indicators
420 */
310struct fast_reg_descriptor { 421struct fast_reg_descriptor {
311 struct list_head list; 422 struct list_head list;
312 /* For fast registration - FRWR */
313 struct ib_mr *data_mr; 423 struct ib_mr *data_mr;
314 struct ib_fast_reg_page_list *data_frpl; 424 struct ib_fast_reg_page_list *data_frpl;
315 struct iser_pi_context *pi_ctx; 425 struct iser_pi_context *pi_ctx;
316 /* registration indicators container */
317 u8 reg_indicators; 426 u8 reg_indicators;
318}; 427};
319 428
429/**
430 * struct ib_conn - Infiniband related objects
431 *
432 * @cma_id: rdma_cm connection maneger handle
433 * @qp: Connection Queue-pair
434 * @post_recv_buf_count: post receive counter
435 * @rx_wr: receive work request for batch posts
436 * @device: reference to iser device
437 * @comp: iser completion context
438 * @pi_support: Indicate device T10-PI support
439 * @beacon: beacon send wr to signal all flush errors were drained
440 * @flush_comp: completes when all connection completions consumed
441 * @lock: protects fmr/fastreg pool
442 * @union.fmr:
443 * @pool: FMR pool for fast registrations
444 * @page_vec: page vector to hold mapped commands pages
445 * used for registration
446 * @union.fastreg:
447 * @pool: Fast registration descriptors pool for fast
448 * registrations
449 * @pool_size: Size of pool
450 */
451struct ib_conn {
452 struct rdma_cm_id *cma_id;
453 struct ib_qp *qp;
454 int post_recv_buf_count;
455 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
456 struct iser_device *device;
457 struct iser_comp *comp;
458 bool pi_support;
459 struct ib_send_wr beacon;
460 struct completion flush_comp;
461 spinlock_t lock;
462 union {
463 struct {
464 struct ib_fmr_pool *pool;
465 struct iser_page_vec *page_vec;
466 } fmr;
467 struct {
468 struct list_head pool;
469 int pool_size;
470 } fastreg;
471 };
472};
473
474/**
475 * struct iser_conn - iSER connection context
476 *
477 * @ib_conn: connection RDMA resources
478 * @iscsi_conn: link to matching iscsi connection
479 * @ep: transport handle
480 * @state: connection logical state
481 * @qp_max_recv_dtos: maximum number of data outs, corresponds
482 * to max number of post recvs
483 * @qp_max_recv_dtos_mask: (qp_max_recv_dtos - 1)
484 * @min_posted_rx: (qp_max_recv_dtos >> 2)
485 * @name: connection peer portal
486 * @release_work: deffered work for release job
487 * @state_mutex: protects iser onnection state
488 * @stop_completion: conn_stop completion
489 * @ib_completion: RDMA cleanup completion
490 * @up_completion: connection establishment completed
491 * (state is ISER_CONN_UP)
492 * @conn_list: entry in ig conn list
493 * @login_buf: login data buffer (stores login parameters)
494 * @login_req_buf: login request buffer
495 * @login_req_dma: login request buffer dma address
496 * @login_resp_buf: login response buffer
497 * @login_resp_dma: login response buffer dma address
498 * @rx_desc_head: head of rx_descs cyclic buffer
499 * @rx_descs: rx buffers array (cyclic buffer)
500 * @num_rx_descs: number of rx descriptors
501 */
320struct iser_conn { 502struct iser_conn {
503 struct ib_conn ib_conn;
321 struct iscsi_conn *iscsi_conn; 504 struct iscsi_conn *iscsi_conn;
322 struct iscsi_endpoint *ep; 505 struct iscsi_endpoint *ep;
323 enum iser_ib_conn_state state; /* rdma connection state */ 506 enum iser_conn_state state;
324 atomic_t refcount; 507 unsigned qp_max_recv_dtos;
325 spinlock_t lock; /* used for state changes */ 508 unsigned qp_max_recv_dtos_mask;
326 struct iser_device *device; /* device context */ 509 unsigned min_posted_rx;
327 struct rdma_cm_id *cma_id; /* CMA ID */
328 struct ib_qp *qp; /* QP */
329 unsigned qp_max_recv_dtos; /* num of rx buffers */
330 unsigned qp_max_recv_dtos_mask; /* above minus 1 */
331 unsigned min_posted_rx; /* qp_max_recv_dtos >> 2 */
332 int post_recv_buf_count; /* posted rx count */
333 atomic_t post_send_buf_count; /* posted tx count */
334 char name[ISER_OBJECT_NAME_SIZE]; 510 char name[ISER_OBJECT_NAME_SIZE];
335 struct work_struct release_work; 511 struct work_struct release_work;
336 struct completion stop_completion;
337 struct mutex state_mutex; 512 struct mutex state_mutex;
338 struct completion flush_completion; 513 struct completion stop_completion;
514 struct completion ib_completion;
339 struct completion up_completion; 515 struct completion up_completion;
340 struct list_head conn_list; /* entry in ig conn list */ 516 struct list_head conn_list;
341 517
342 char *login_buf; 518 char *login_buf;
343 char *login_req_buf, *login_resp_buf; 519 char *login_req_buf, *login_resp_buf;
344 u64 login_req_dma, login_resp_dma; 520 u64 login_req_dma, login_resp_dma;
345 unsigned int rx_desc_head; 521 unsigned int rx_desc_head;
346 struct iser_rx_desc *rx_descs; 522 struct iser_rx_desc *rx_descs;
347 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX]; 523 u32 num_rx_descs;
348 bool pi_support;
349
350 /* Connection memory registration pool */
351 union {
352 struct {
353 struct ib_fmr_pool *pool; /* pool of IB FMRs */
354 struct iser_page_vec *page_vec; /* represents SG to fmr maps*
355 * maps serialized as tx is*/
356 } fmr;
357 struct {
358 struct list_head pool;
359 int pool_size;
360 } fastreg;
361 };
362}; 524};
363 525
526/**
527 * struct iscsi_iser_task - iser task context
528 *
529 * @desc: TX descriptor
530 * @iser_conn: link to iser connection
531 * @status: current task status
532 * @sc: link to scsi command
533 * @command_sent: indicate if command was sent
534 * @dir: iser data direction
535 * @rdma_regd: task rdma registration desc
536 * @data: iser data buffer desc
537 * @data_copy: iser data copy buffer desc (bounce buffer)
538 * @prot: iser protection buffer desc
539 * @prot_copy: iser protection copy buffer desc (bounce buffer)
540 */
364struct iscsi_iser_task { 541struct iscsi_iser_task {
365 struct iser_tx_desc desc; 542 struct iser_tx_desc desc;
366 struct iser_conn *ib_conn; 543 struct iser_conn *iser_conn;
367 enum iser_task_status status; 544 enum iser_task_status status;
368 struct scsi_cmnd *sc; 545 struct scsi_cmnd *sc;
369 int command_sent; /* set if command sent */ 546 int command_sent;
370 int dir[ISER_DIRS_NUM]; /* set if dir use*/ 547 int dir[ISER_DIRS_NUM];
371 struct iser_regd_buf rdma_regd[ISER_DIRS_NUM];/* regd rdma buf */ 548 struct iser_regd_buf rdma_regd[ISER_DIRS_NUM];
372 struct iser_data_buf data[ISER_DIRS_NUM]; /* orig. data des*/ 549 struct iser_data_buf data[ISER_DIRS_NUM];
373 struct iser_data_buf data_copy[ISER_DIRS_NUM];/* contig. copy */ 550 struct iser_data_buf data_copy[ISER_DIRS_NUM];
374 struct iser_data_buf prot[ISER_DIRS_NUM]; /* prot desc */ 551 struct iser_data_buf prot[ISER_DIRS_NUM];
375 struct iser_data_buf prot_copy[ISER_DIRS_NUM];/* prot copy */ 552 struct iser_data_buf prot_copy[ISER_DIRS_NUM];
376}; 553};
377 554
378struct iser_page_vec { 555struct iser_page_vec {
@@ -382,17 +559,20 @@ struct iser_page_vec {
382 int data_size; 559 int data_size;
383}; 560};
384 561
385struct iser_cq_desc { 562/**
386 struct iser_device *device; 563 * struct iser_global: iSER global context
387 int cq_index; 564 *
388}; 565 * @device_list_mutex: protects device_list
389 566 * @device_list: iser devices global list
567 * @connlist_mutex: protects connlist
568 * @connlist: iser connections global list
569 * @desc_cache: kmem cache for tx dataout
570 */
390struct iser_global { 571struct iser_global {
391 struct mutex device_list_mutex;/* */ 572 struct mutex device_list_mutex;
392 struct list_head device_list; /* all iSER devices */ 573 struct list_head device_list;
393 struct mutex connlist_mutex; 574 struct mutex connlist_mutex;
394 struct list_head connlist; /* all iSER IB connections */ 575 struct list_head connlist;
395
396 struct kmem_cache *desc_cache; 576 struct kmem_cache *desc_cache;
397}; 577};
398 578
@@ -401,9 +581,6 @@ extern int iser_debug_level;
401extern bool iser_pi_enable; 581extern bool iser_pi_enable;
402extern int iser_pi_guard; 582extern int iser_pi_guard;
403 583
404/* allocate connection resources needed for rdma functionality */
405int iser_conn_set_full_featured_mode(struct iscsi_conn *conn);
406
407int iser_send_control(struct iscsi_conn *conn, 584int iser_send_control(struct iscsi_conn *conn,
408 struct iscsi_task *task); 585 struct iscsi_task *task);
409 586
@@ -415,29 +592,30 @@ int iser_send_data_out(struct iscsi_conn *conn,
415 struct iscsi_data *hdr); 592 struct iscsi_data *hdr);
416 593
417void iscsi_iser_recv(struct iscsi_conn *conn, 594void iscsi_iser_recv(struct iscsi_conn *conn,
418 struct iscsi_hdr *hdr, 595 struct iscsi_hdr *hdr,
419 char *rx_data, 596 char *rx_data,
420 int rx_data_len); 597 int rx_data_len);
421 598
422void iser_conn_init(struct iser_conn *ib_conn); 599void iser_conn_init(struct iser_conn *iser_conn);
423 600
424void iser_conn_release(struct iser_conn *ib_conn); 601void iser_conn_release(struct iser_conn *iser_conn);
425 602
426void iser_conn_terminate(struct iser_conn *ib_conn); 603int iser_conn_terminate(struct iser_conn *iser_conn);
427 604
428void iser_release_work(struct work_struct *work); 605void iser_release_work(struct work_struct *work);
429 606
430void iser_rcv_completion(struct iser_rx_desc *desc, 607void iser_rcv_completion(struct iser_rx_desc *desc,
431 unsigned long dto_xfer_len, 608 unsigned long dto_xfer_len,
432 struct iser_conn *ib_conn); 609 struct ib_conn *ib_conn);
433 610
434void iser_snd_completion(struct iser_tx_desc *desc, struct iser_conn *ib_conn); 611void iser_snd_completion(struct iser_tx_desc *desc,
612 struct ib_conn *ib_conn);
435 613
436void iser_task_rdma_init(struct iscsi_iser_task *task); 614void iser_task_rdma_init(struct iscsi_iser_task *task);
437 615
438void iser_task_rdma_finalize(struct iscsi_iser_task *task); 616void iser_task_rdma_finalize(struct iscsi_iser_task *task);
439 617
440void iser_free_rx_descriptors(struct iser_conn *ib_conn); 618void iser_free_rx_descriptors(struct iser_conn *iser_conn);
441 619
442void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 620void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
443 struct iser_data_buf *mem, 621 struct iser_data_buf *mem,
@@ -449,38 +627,40 @@ int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *task,
449int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task, 627int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task,
450 enum iser_data_dir cmd_dir); 628 enum iser_data_dir cmd_dir);
451 629
452int iser_connect(struct iser_conn *ib_conn, 630int iser_connect(struct iser_conn *iser_conn,
453 struct sockaddr *src_addr, 631 struct sockaddr *src_addr,
454 struct sockaddr *dst_addr, 632 struct sockaddr *dst_addr,
455 int non_blocking); 633 int non_blocking);
456 634
457int iser_reg_page_vec(struct iser_conn *ib_conn, 635int iser_reg_page_vec(struct ib_conn *ib_conn,
458 struct iser_page_vec *page_vec, 636 struct iser_page_vec *page_vec,
459 struct iser_mem_reg *mem_reg); 637 struct iser_mem_reg *mem_reg);
460 638
461void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task, 639void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
462 enum iser_data_dir cmd_dir); 640 enum iser_data_dir cmd_dir);
463void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, 641void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
464 enum iser_data_dir cmd_dir); 642 enum iser_data_dir cmd_dir);
465 643
466int iser_post_recvl(struct iser_conn *ib_conn); 644int iser_post_recvl(struct iser_conn *iser_conn);
467int iser_post_recvm(struct iser_conn *ib_conn, int count); 645int iser_post_recvm(struct iser_conn *iser_conn, int count);
468int iser_post_send(struct iser_conn *ib_conn, struct iser_tx_desc *tx_desc); 646int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
647 bool signal);
469 648
470int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 649int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
471 struct iser_data_buf *data, 650 struct iser_data_buf *data,
472 enum iser_data_dir iser_dir, 651 enum iser_data_dir iser_dir,
473 enum dma_data_direction dma_dir); 652 enum dma_data_direction dma_dir);
474 653
475void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 654void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
476 struct iser_data_buf *data); 655 struct iser_data_buf *data);
477int iser_initialize_task_headers(struct iscsi_task *task, 656int iser_initialize_task_headers(struct iscsi_task *task,
478 struct iser_tx_desc *tx_desc); 657 struct iser_tx_desc *tx_desc);
479int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session); 658int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
480int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max); 659 struct iscsi_session *session);
481void iser_free_fmr_pool(struct iser_conn *ib_conn); 660int iser_create_fmr_pool(struct ib_conn *ib_conn, unsigned cmds_max);
482int iser_create_fastreg_pool(struct iser_conn *ib_conn, unsigned cmds_max); 661void iser_free_fmr_pool(struct ib_conn *ib_conn);
483void iser_free_fastreg_pool(struct iser_conn *ib_conn); 662int iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max);
663void iser_free_fastreg_pool(struct ib_conn *ib_conn);
484u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task, 664u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
485 enum iser_data_dir cmd_dir, sector_t *sector); 665 enum iser_data_dir cmd_dir, sector_t *sector);
486#endif 666#endif
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index 8d44a4060634..5a489ea63732 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -49,7 +49,7 @@ static int iser_prepare_read_cmd(struct iscsi_task *task)
49 49
50{ 50{
51 struct iscsi_iser_task *iser_task = task->dd_data; 51 struct iscsi_iser_task *iser_task = task->dd_data;
52 struct iser_device *device = iser_task->ib_conn->device; 52 struct iser_device *device = iser_task->iser_conn->ib_conn.device;
53 struct iser_regd_buf *regd_buf; 53 struct iser_regd_buf *regd_buf;
54 int err; 54 int err;
55 struct iser_hdr *hdr = &iser_task->desc.iser_header; 55 struct iser_hdr *hdr = &iser_task->desc.iser_header;
@@ -103,7 +103,7 @@ iser_prepare_write_cmd(struct iscsi_task *task,
103 unsigned int edtl) 103 unsigned int edtl)
104{ 104{
105 struct iscsi_iser_task *iser_task = task->dd_data; 105 struct iscsi_iser_task *iser_task = task->dd_data;
106 struct iser_device *device = iser_task->ib_conn->device; 106 struct iser_device *device = iser_task->iser_conn->ib_conn.device;
107 struct iser_regd_buf *regd_buf; 107 struct iser_regd_buf *regd_buf;
108 int err; 108 int err;
109 struct iser_hdr *hdr = &iser_task->desc.iser_header; 109 struct iser_hdr *hdr = &iser_task->desc.iser_header;
@@ -160,10 +160,10 @@ iser_prepare_write_cmd(struct iscsi_task *task,
160} 160}
161 161
162/* creates a new tx descriptor and adds header regd buffer */ 162/* creates a new tx descriptor and adds header regd buffer */
163static void iser_create_send_desc(struct iser_conn *ib_conn, 163static void iser_create_send_desc(struct iser_conn *iser_conn,
164 struct iser_tx_desc *tx_desc) 164 struct iser_tx_desc *tx_desc)
165{ 165{
166 struct iser_device *device = ib_conn->device; 166 struct iser_device *device = iser_conn->ib_conn.device;
167 167
168 ib_dma_sync_single_for_cpu(device->ib_device, 168 ib_dma_sync_single_for_cpu(device->ib_device,
169 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 169 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
@@ -179,103 +179,108 @@ static void iser_create_send_desc(struct iser_conn *ib_conn,
179 } 179 }
180} 180}
181 181
182static void iser_free_login_buf(struct iser_conn *ib_conn) 182static void iser_free_login_buf(struct iser_conn *iser_conn)
183{ 183{
184 if (!ib_conn->login_buf) 184 struct iser_device *device = iser_conn->ib_conn.device;
185
186 if (!iser_conn->login_buf)
185 return; 187 return;
186 188
187 if (ib_conn->login_req_dma) 189 if (iser_conn->login_req_dma)
188 ib_dma_unmap_single(ib_conn->device->ib_device, 190 ib_dma_unmap_single(device->ib_device,
189 ib_conn->login_req_dma, 191 iser_conn->login_req_dma,
190 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); 192 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
191 193
192 if (ib_conn->login_resp_dma) 194 if (iser_conn->login_resp_dma)
193 ib_dma_unmap_single(ib_conn->device->ib_device, 195 ib_dma_unmap_single(device->ib_device,
194 ib_conn->login_resp_dma, 196 iser_conn->login_resp_dma,
195 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); 197 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
196 198
197 kfree(ib_conn->login_buf); 199 kfree(iser_conn->login_buf);
198 200
199 /* make sure we never redo any unmapping */ 201 /* make sure we never redo any unmapping */
200 ib_conn->login_req_dma = 0; 202 iser_conn->login_req_dma = 0;
201 ib_conn->login_resp_dma = 0; 203 iser_conn->login_resp_dma = 0;
202 ib_conn->login_buf = NULL; 204 iser_conn->login_buf = NULL;
203} 205}
204 206
205static int iser_alloc_login_buf(struct iser_conn *ib_conn) 207static int iser_alloc_login_buf(struct iser_conn *iser_conn)
206{ 208{
207 struct iser_device *device; 209 struct iser_device *device = iser_conn->ib_conn.device;
208 int req_err, resp_err; 210 int req_err, resp_err;
209 211
210 BUG_ON(ib_conn->device == NULL); 212 BUG_ON(device == NULL);
211 213
212 device = ib_conn->device; 214 iser_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
213
214 ib_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
215 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 215 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
216 if (!ib_conn->login_buf) 216 if (!iser_conn->login_buf)
217 goto out_err; 217 goto out_err;
218 218
219 ib_conn->login_req_buf = ib_conn->login_buf; 219 iser_conn->login_req_buf = iser_conn->login_buf;
220 ib_conn->login_resp_buf = ib_conn->login_buf + 220 iser_conn->login_resp_buf = iser_conn->login_buf +
221 ISCSI_DEF_MAX_RECV_SEG_LEN; 221 ISCSI_DEF_MAX_RECV_SEG_LEN;
222 222
223 ib_conn->login_req_dma = ib_dma_map_single(ib_conn->device->ib_device, 223 iser_conn->login_req_dma = ib_dma_map_single(device->ib_device,
224 (void *)ib_conn->login_req_buf, 224 iser_conn->login_req_buf,
225 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); 225 ISCSI_DEF_MAX_RECV_SEG_LEN,
226 DMA_TO_DEVICE);
226 227
227 ib_conn->login_resp_dma = ib_dma_map_single(ib_conn->device->ib_device, 228 iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device,
228 (void *)ib_conn->login_resp_buf, 229 iser_conn->login_resp_buf,
229 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); 230 ISER_RX_LOGIN_SIZE,
231 DMA_FROM_DEVICE);
230 232
231 req_err = ib_dma_mapping_error(device->ib_device, 233 req_err = ib_dma_mapping_error(device->ib_device,
232 ib_conn->login_req_dma); 234 iser_conn->login_req_dma);
233 resp_err = ib_dma_mapping_error(device->ib_device, 235 resp_err = ib_dma_mapping_error(device->ib_device,
234 ib_conn->login_resp_dma); 236 iser_conn->login_resp_dma);
235 237
236 if (req_err || resp_err) { 238 if (req_err || resp_err) {
237 if (req_err) 239 if (req_err)
238 ib_conn->login_req_dma = 0; 240 iser_conn->login_req_dma = 0;
239 if (resp_err) 241 if (resp_err)
240 ib_conn->login_resp_dma = 0; 242 iser_conn->login_resp_dma = 0;
241 goto free_login_buf; 243 goto free_login_buf;
242 } 244 }
243 return 0; 245 return 0;
244 246
245free_login_buf: 247free_login_buf:
246 iser_free_login_buf(ib_conn); 248 iser_free_login_buf(iser_conn);
247 249
248out_err: 250out_err:
249 iser_err("unable to alloc or map login buf\n"); 251 iser_err("unable to alloc or map login buf\n");
250 return -ENOMEM; 252 return -ENOMEM;
251} 253}
252 254
253int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session) 255int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
256 struct iscsi_session *session)
254{ 257{
255 int i, j; 258 int i, j;
256 u64 dma_addr; 259 u64 dma_addr;
257 struct iser_rx_desc *rx_desc; 260 struct iser_rx_desc *rx_desc;
258 struct ib_sge *rx_sg; 261 struct ib_sge *rx_sg;
259 struct iser_device *device = ib_conn->device; 262 struct ib_conn *ib_conn = &iser_conn->ib_conn;
263 struct iser_device *device = ib_conn->device;
260 264
261 ib_conn->qp_max_recv_dtos = session->cmds_max; 265 iser_conn->qp_max_recv_dtos = session->cmds_max;
262 ib_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */ 266 iser_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */
263 ib_conn->min_posted_rx = ib_conn->qp_max_recv_dtos >> 2; 267 iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;
264 268
265 if (device->iser_alloc_rdma_reg_res(ib_conn, session->scsi_cmds_max)) 269 if (device->iser_alloc_rdma_reg_res(ib_conn, session->scsi_cmds_max))
266 goto create_rdma_reg_res_failed; 270 goto create_rdma_reg_res_failed;
267 271
268 if (iser_alloc_login_buf(ib_conn)) 272 if (iser_alloc_login_buf(iser_conn))
269 goto alloc_login_buf_fail; 273 goto alloc_login_buf_fail;
270 274
271 ib_conn->rx_descs = kmalloc(session->cmds_max * 275 iser_conn->num_rx_descs = session->cmds_max;
276 iser_conn->rx_descs = kmalloc(iser_conn->num_rx_descs *
272 sizeof(struct iser_rx_desc), GFP_KERNEL); 277 sizeof(struct iser_rx_desc), GFP_KERNEL);
273 if (!ib_conn->rx_descs) 278 if (!iser_conn->rx_descs)
274 goto rx_desc_alloc_fail; 279 goto rx_desc_alloc_fail;
275 280
276 rx_desc = ib_conn->rx_descs; 281 rx_desc = iser_conn->rx_descs;
277 282
278 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) { 283 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) {
279 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, 284 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
280 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 285 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
281 if (ib_dma_mapping_error(device->ib_device, dma_addr)) 286 if (ib_dma_mapping_error(device->ib_device, dma_addr))
@@ -289,18 +294,18 @@ int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *s
289 rx_sg->lkey = device->mr->lkey; 294 rx_sg->lkey = device->mr->lkey;
290 } 295 }
291 296
292 ib_conn->rx_desc_head = 0; 297 iser_conn->rx_desc_head = 0;
293 return 0; 298 return 0;
294 299
295rx_desc_dma_map_failed: 300rx_desc_dma_map_failed:
296 rx_desc = ib_conn->rx_descs; 301 rx_desc = iser_conn->rx_descs;
297 for (j = 0; j < i; j++, rx_desc++) 302 for (j = 0; j < i; j++, rx_desc++)
298 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 303 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
299 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 304 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
300 kfree(ib_conn->rx_descs); 305 kfree(iser_conn->rx_descs);
301 ib_conn->rx_descs = NULL; 306 iser_conn->rx_descs = NULL;
302rx_desc_alloc_fail: 307rx_desc_alloc_fail:
303 iser_free_login_buf(ib_conn); 308 iser_free_login_buf(iser_conn);
304alloc_login_buf_fail: 309alloc_login_buf_fail:
305 device->iser_free_rdma_reg_res(ib_conn); 310 device->iser_free_rdma_reg_res(ib_conn);
306create_rdma_reg_res_failed: 311create_rdma_reg_res_failed:
@@ -308,33 +313,35 @@ create_rdma_reg_res_failed:
308 return -ENOMEM; 313 return -ENOMEM;
309} 314}
310 315
311void iser_free_rx_descriptors(struct iser_conn *ib_conn) 316void iser_free_rx_descriptors(struct iser_conn *iser_conn)
312{ 317{
313 int i; 318 int i;
314 struct iser_rx_desc *rx_desc; 319 struct iser_rx_desc *rx_desc;
320 struct ib_conn *ib_conn = &iser_conn->ib_conn;
315 struct iser_device *device = ib_conn->device; 321 struct iser_device *device = ib_conn->device;
316 322
317 if (!ib_conn->rx_descs) 323 if (!iser_conn->rx_descs)
318 goto free_login_buf; 324 goto free_login_buf;
319 325
320 if (device->iser_free_rdma_reg_res) 326 if (device->iser_free_rdma_reg_res)
321 device->iser_free_rdma_reg_res(ib_conn); 327 device->iser_free_rdma_reg_res(ib_conn);
322 328
323 rx_desc = ib_conn->rx_descs; 329 rx_desc = iser_conn->rx_descs;
324 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) 330 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)
325 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 331 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
326 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 332 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
327 kfree(ib_conn->rx_descs); 333 kfree(iser_conn->rx_descs);
328 /* make sure we never redo any unmapping */ 334 /* make sure we never redo any unmapping */
329 ib_conn->rx_descs = NULL; 335 iser_conn->rx_descs = NULL;
330 336
331free_login_buf: 337free_login_buf:
332 iser_free_login_buf(ib_conn); 338 iser_free_login_buf(iser_conn);
333} 339}
334 340
335static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) 341static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)
336{ 342{
337 struct iser_conn *ib_conn = conn->dd_data; 343 struct iser_conn *iser_conn = conn->dd_data;
344 struct ib_conn *ib_conn = &iser_conn->ib_conn;
338 struct iscsi_session *session = conn->session; 345 struct iscsi_session *session = conn->session;
339 346
340 iser_dbg("req op %x flags %x\n", req->opcode, req->flags); 347 iser_dbg("req op %x flags %x\n", req->opcode, req->flags);
@@ -343,34 +350,37 @@ static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)
343 return 0; 350 return 0;
344 351
345 /* 352 /*
346 * Check that there is one posted recv buffer (for the last login 353 * Check that there is one posted recv buffer
347 * response) and no posted send buffers left - they must have been 354 * (for the last login response).
348 * consumed during previous login phases.
349 */ 355 */
350 WARN_ON(ib_conn->post_recv_buf_count != 1); 356 WARN_ON(ib_conn->post_recv_buf_count != 1);
351 WARN_ON(atomic_read(&ib_conn->post_send_buf_count) != 0);
352 357
353 if (session->discovery_sess) { 358 if (session->discovery_sess) {
354 iser_info("Discovery session, re-using login RX buffer\n"); 359 iser_info("Discovery session, re-using login RX buffer\n");
355 return 0; 360 return 0;
356 } else 361 } else
357 iser_info("Normal session, posting batch of RX %d buffers\n", 362 iser_info("Normal session, posting batch of RX %d buffers\n",
358 ib_conn->min_posted_rx); 363 iser_conn->min_posted_rx);
359 364
360 /* Initial post receive buffers */ 365 /* Initial post receive buffers */
361 if (iser_post_recvm(ib_conn, ib_conn->min_posted_rx)) 366 if (iser_post_recvm(iser_conn, iser_conn->min_posted_rx))
362 return -ENOMEM; 367 return -ENOMEM;
363 368
364 return 0; 369 return 0;
365} 370}
366 371
372static inline bool iser_signal_comp(int sig_count)
373{
374 return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0);
375}
376
367/** 377/**
368 * iser_send_command - send command PDU 378 * iser_send_command - send command PDU
369 */ 379 */
370int iser_send_command(struct iscsi_conn *conn, 380int iser_send_command(struct iscsi_conn *conn,
371 struct iscsi_task *task) 381 struct iscsi_task *task)
372{ 382{
373 struct iser_conn *ib_conn = conn->dd_data; 383 struct iser_conn *iser_conn = conn->dd_data;
374 struct iscsi_iser_task *iser_task = task->dd_data; 384 struct iscsi_iser_task *iser_task = task->dd_data;
375 unsigned long edtl; 385 unsigned long edtl;
376 int err; 386 int err;
@@ -378,12 +388,13 @@ int iser_send_command(struct iscsi_conn *conn,
378 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; 388 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;
379 struct scsi_cmnd *sc = task->sc; 389 struct scsi_cmnd *sc = task->sc;
380 struct iser_tx_desc *tx_desc = &iser_task->desc; 390 struct iser_tx_desc *tx_desc = &iser_task->desc;
391 static unsigned sig_count;
381 392
382 edtl = ntohl(hdr->data_length); 393 edtl = ntohl(hdr->data_length);
383 394
384 /* build the tx desc regd header and add it to the tx desc dto */ 395 /* build the tx desc regd header and add it to the tx desc dto */
385 tx_desc->type = ISCSI_TX_SCSI_COMMAND; 396 tx_desc->type = ISCSI_TX_SCSI_COMMAND;
386 iser_create_send_desc(ib_conn, tx_desc); 397 iser_create_send_desc(iser_conn, tx_desc);
387 398
388 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 399 if (hdr->flags & ISCSI_FLAG_CMD_READ) {
389 data_buf = &iser_task->data[ISER_DIR_IN]; 400 data_buf = &iser_task->data[ISER_DIR_IN];
@@ -423,7 +434,8 @@ int iser_send_command(struct iscsi_conn *conn,
423 434
424 iser_task->status = ISER_TASK_STATUS_STARTED; 435 iser_task->status = ISER_TASK_STATUS_STARTED;
425 436
426 err = iser_post_send(ib_conn, tx_desc); 437 err = iser_post_send(&iser_conn->ib_conn, tx_desc,
438 iser_signal_comp(++sig_count));
427 if (!err) 439 if (!err)
428 return 0; 440 return 0;
429 441
@@ -439,7 +451,7 @@ int iser_send_data_out(struct iscsi_conn *conn,
439 struct iscsi_task *task, 451 struct iscsi_task *task,
440 struct iscsi_data *hdr) 452 struct iscsi_data *hdr)
441{ 453{
442 struct iser_conn *ib_conn = conn->dd_data; 454 struct iser_conn *iser_conn = conn->dd_data;
443 struct iscsi_iser_task *iser_task = task->dd_data; 455 struct iscsi_iser_task *iser_task = task->dd_data;
444 struct iser_tx_desc *tx_desc = NULL; 456 struct iser_tx_desc *tx_desc = NULL;
445 struct iser_regd_buf *regd_buf; 457 struct iser_regd_buf *regd_buf;
@@ -488,7 +500,7 @@ int iser_send_data_out(struct iscsi_conn *conn,
488 itt, buf_offset, data_seg_len); 500 itt, buf_offset, data_seg_len);
489 501
490 502
491 err = iser_post_send(ib_conn, tx_desc); 503 err = iser_post_send(&iser_conn->ib_conn, tx_desc, true);
492 if (!err) 504 if (!err)
493 return 0; 505 return 0;
494 506
@@ -501,7 +513,7 @@ send_data_out_error:
501int iser_send_control(struct iscsi_conn *conn, 513int iser_send_control(struct iscsi_conn *conn,
502 struct iscsi_task *task) 514 struct iscsi_task *task)
503{ 515{
504 struct iser_conn *ib_conn = conn->dd_data; 516 struct iser_conn *iser_conn = conn->dd_data;
505 struct iscsi_iser_task *iser_task = task->dd_data; 517 struct iscsi_iser_task *iser_task = task->dd_data;
506 struct iser_tx_desc *mdesc = &iser_task->desc; 518 struct iser_tx_desc *mdesc = &iser_task->desc;
507 unsigned long data_seg_len; 519 unsigned long data_seg_len;
@@ -510,9 +522,9 @@ int iser_send_control(struct iscsi_conn *conn,
510 522
511 /* build the tx desc regd header and add it to the tx desc dto */ 523 /* build the tx desc regd header and add it to the tx desc dto */
512 mdesc->type = ISCSI_TX_CONTROL; 524 mdesc->type = ISCSI_TX_CONTROL;
513 iser_create_send_desc(ib_conn, mdesc); 525 iser_create_send_desc(iser_conn, mdesc);
514 526
515 device = ib_conn->device; 527 device = iser_conn->ib_conn.device;
516 528
517 data_seg_len = ntoh24(task->hdr->dlength); 529 data_seg_len = ntoh24(task->hdr->dlength);
518 530
@@ -524,16 +536,16 @@ int iser_send_control(struct iscsi_conn *conn,
524 } 536 }
525 537
526 ib_dma_sync_single_for_cpu(device->ib_device, 538 ib_dma_sync_single_for_cpu(device->ib_device,
527 ib_conn->login_req_dma, task->data_count, 539 iser_conn->login_req_dma, task->data_count,
528 DMA_TO_DEVICE); 540 DMA_TO_DEVICE);
529 541
530 memcpy(ib_conn->login_req_buf, task->data, task->data_count); 542 memcpy(iser_conn->login_req_buf, task->data, task->data_count);
531 543
532 ib_dma_sync_single_for_device(device->ib_device, 544 ib_dma_sync_single_for_device(device->ib_device,
533 ib_conn->login_req_dma, task->data_count, 545 iser_conn->login_req_dma, task->data_count,
534 DMA_TO_DEVICE); 546 DMA_TO_DEVICE);
535 547
536 tx_dsg->addr = ib_conn->login_req_dma; 548 tx_dsg->addr = iser_conn->login_req_dma;
537 tx_dsg->length = task->data_count; 549 tx_dsg->length = task->data_count;
538 tx_dsg->lkey = device->mr->lkey; 550 tx_dsg->lkey = device->mr->lkey;
539 mdesc->num_sge = 2; 551 mdesc->num_sge = 2;
@@ -542,7 +554,7 @@ int iser_send_control(struct iscsi_conn *conn,
542 if (task == conn->login_task) { 554 if (task == conn->login_task) {
543 iser_dbg("op %x dsl %lx, posting login rx buffer\n", 555 iser_dbg("op %x dsl %lx, posting login rx buffer\n",
544 task->hdr->opcode, data_seg_len); 556 task->hdr->opcode, data_seg_len);
545 err = iser_post_recvl(ib_conn); 557 err = iser_post_recvl(iser_conn);
546 if (err) 558 if (err)
547 goto send_control_error; 559 goto send_control_error;
548 err = iser_post_rx_bufs(conn, task->hdr); 560 err = iser_post_rx_bufs(conn, task->hdr);
@@ -550,7 +562,7 @@ int iser_send_control(struct iscsi_conn *conn,
550 goto send_control_error; 562 goto send_control_error;
551 } 563 }
552 564
553 err = iser_post_send(ib_conn, mdesc); 565 err = iser_post_send(&iser_conn->ib_conn, mdesc, true);
554 if (!err) 566 if (!err)
555 return 0; 567 return 0;
556 568
@@ -564,15 +576,17 @@ send_control_error:
564 */ 576 */
565void iser_rcv_completion(struct iser_rx_desc *rx_desc, 577void iser_rcv_completion(struct iser_rx_desc *rx_desc,
566 unsigned long rx_xfer_len, 578 unsigned long rx_xfer_len,
567 struct iser_conn *ib_conn) 579 struct ib_conn *ib_conn)
568{ 580{
581 struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
582 ib_conn);
569 struct iscsi_hdr *hdr; 583 struct iscsi_hdr *hdr;
570 u64 rx_dma; 584 u64 rx_dma;
571 int rx_buflen, outstanding, count, err; 585 int rx_buflen, outstanding, count, err;
572 586
573 /* differentiate between login to all other PDUs */ 587 /* differentiate between login to all other PDUs */
574 if ((char *)rx_desc == ib_conn->login_resp_buf) { 588 if ((char *)rx_desc == iser_conn->login_resp_buf) {
575 rx_dma = ib_conn->login_resp_dma; 589 rx_dma = iser_conn->login_resp_dma;
576 rx_buflen = ISER_RX_LOGIN_SIZE; 590 rx_buflen = ISER_RX_LOGIN_SIZE;
577 } else { 591 } else {
578 rx_dma = rx_desc->dma_addr; 592 rx_dma = rx_desc->dma_addr;
@@ -580,14 +594,14 @@ void iser_rcv_completion(struct iser_rx_desc *rx_desc,
580 } 594 }
581 595
582 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma, 596 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
583 rx_buflen, DMA_FROM_DEVICE); 597 rx_buflen, DMA_FROM_DEVICE);
584 598
585 hdr = &rx_desc->iscsi_header; 599 hdr = &rx_desc->iscsi_header;
586 600
587 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, 601 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode,
588 hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN)); 602 hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN));
589 603
590 iscsi_iser_recv(ib_conn->iscsi_conn, hdr, rx_desc->data, 604 iscsi_iser_recv(iser_conn->iscsi_conn, hdr, rx_desc->data,
591 rx_xfer_len - ISER_HEADERS_LEN); 605 rx_xfer_len - ISER_HEADERS_LEN);
592 606
593 ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma, 607 ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
@@ -599,21 +613,21 @@ void iser_rcv_completion(struct iser_rx_desc *rx_desc,
599 * for the posted rx bufs refcount to become zero handles everything */ 613 * for the posted rx bufs refcount to become zero handles everything */
600 ib_conn->post_recv_buf_count--; 614 ib_conn->post_recv_buf_count--;
601 615
602 if (rx_dma == ib_conn->login_resp_dma) 616 if (rx_dma == iser_conn->login_resp_dma)
603 return; 617 return;
604 618
605 outstanding = ib_conn->post_recv_buf_count; 619 outstanding = ib_conn->post_recv_buf_count;
606 if (outstanding + ib_conn->min_posted_rx <= ib_conn->qp_max_recv_dtos) { 620 if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) {
607 count = min(ib_conn->qp_max_recv_dtos - outstanding, 621 count = min(iser_conn->qp_max_recv_dtos - outstanding,
608 ib_conn->min_posted_rx); 622 iser_conn->min_posted_rx);
609 err = iser_post_recvm(ib_conn, count); 623 err = iser_post_recvm(iser_conn, count);
610 if (err) 624 if (err)
611 iser_err("posting %d rx bufs err %d\n", count, err); 625 iser_err("posting %d rx bufs err %d\n", count, err);
612 } 626 }
613} 627}
614 628
615void iser_snd_completion(struct iser_tx_desc *tx_desc, 629void iser_snd_completion(struct iser_tx_desc *tx_desc,
616 struct iser_conn *ib_conn) 630 struct ib_conn *ib_conn)
617{ 631{
618 struct iscsi_task *task; 632 struct iscsi_task *task;
619 struct iser_device *device = ib_conn->device; 633 struct iser_device *device = ib_conn->device;
@@ -625,8 +639,6 @@ void iser_snd_completion(struct iser_tx_desc *tx_desc,
625 tx_desc = NULL; 639 tx_desc = NULL;
626 } 640 }
627 641
628 atomic_dec(&ib_conn->post_send_buf_count);
629
630 if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL) { 642 if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL) {
631 /* this arithmetic is legal by libiscsi dd_data allocation */ 643 /* this arithmetic is legal by libiscsi dd_data allocation */
632 task = (void *) ((long)(void *)tx_desc - 644 task = (void *) ((long)(void *)tx_desc -
@@ -658,7 +670,7 @@ void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
658 670
659void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) 671void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
660{ 672{
661 struct iser_device *device = iser_task->ib_conn->device; 673 struct iser_device *device = iser_task->iser_conn->ib_conn.device;
662 int is_rdma_data_aligned = 1; 674 int is_rdma_data_aligned = 1;
663 int is_rdma_prot_aligned = 1; 675 int is_rdma_prot_aligned = 1;
664 int prot_count = scsi_prot_sg_count(iser_task->sc); 676 int prot_count = scsi_prot_sg_count(iser_task->sc);
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index 47acd3ad3a17..6c5ce357fba6 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -49,7 +49,7 @@ static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
49 struct iser_data_buf *data_copy, 49 struct iser_data_buf *data_copy,
50 enum iser_data_dir cmd_dir) 50 enum iser_data_dir cmd_dir)
51{ 51{
52 struct ib_device *dev = iser_task->ib_conn->device->ib_device; 52 struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device;
53 struct scatterlist *sgl = (struct scatterlist *)data->buf; 53 struct scatterlist *sgl = (struct scatterlist *)data->buf;
54 struct scatterlist *sg; 54 struct scatterlist *sg;
55 char *mem = NULL; 55 char *mem = NULL;
@@ -116,7 +116,7 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
116 struct ib_device *dev; 116 struct ib_device *dev;
117 unsigned long cmd_data_len; 117 unsigned long cmd_data_len;
118 118
119 dev = iser_task->ib_conn->device->ib_device; 119 dev = iser_task->iser_conn->ib_conn.device->ib_device;
120 120
121 ib_dma_unmap_sg(dev, &data_copy->sg_single, 1, 121 ib_dma_unmap_sg(dev, &data_copy->sg_single, 1,
122 (cmd_dir == ISER_DIR_OUT) ? 122 (cmd_dir == ISER_DIR_OUT) ?
@@ -322,7 +322,7 @@ int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
322 struct ib_device *dev; 322 struct ib_device *dev;
323 323
324 iser_task->dir[iser_dir] = 1; 324 iser_task->dir[iser_dir] = 1;
325 dev = iser_task->ib_conn->device->ib_device; 325 dev = iser_task->iser_conn->ib_conn.device->ib_device;
326 326
327 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); 327 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir);
328 if (data->dma_nents == 0) { 328 if (data->dma_nents == 0) {
@@ -337,7 +337,7 @@ void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
337{ 337{
338 struct ib_device *dev; 338 struct ib_device *dev;
339 339
340 dev = iser_task->ib_conn->device->ib_device; 340 dev = iser_task->iser_conn->ib_conn.device->ib_device;
341 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE); 341 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE);
342} 342}
343 343
@@ -348,7 +348,7 @@ static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task,
348 enum iser_data_dir cmd_dir, 348 enum iser_data_dir cmd_dir,
349 int aligned_len) 349 int aligned_len)
350{ 350{
351 struct iscsi_conn *iscsi_conn = iser_task->ib_conn->iscsi_conn; 351 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn;
352 352
353 iscsi_conn->fmr_unalign_cnt++; 353 iscsi_conn->fmr_unalign_cnt++;
354 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", 354 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n",
@@ -377,7 +377,7 @@ static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task,
377int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task, 377int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
378 enum iser_data_dir cmd_dir) 378 enum iser_data_dir cmd_dir)
379{ 379{
380 struct iser_conn *ib_conn = iser_task->ib_conn; 380 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
381 struct iser_device *device = ib_conn->device; 381 struct iser_device *device = ib_conn->device;
382 struct ib_device *ibdev = device->ib_device; 382 struct ib_device *ibdev = device->ib_device;
383 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 383 struct iser_data_buf *mem = &iser_task->data[cmd_dir];
@@ -432,7 +432,7 @@ int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
432 ib_conn->fmr.page_vec->offset); 432 ib_conn->fmr.page_vec->offset);
433 for (i = 0; i < ib_conn->fmr.page_vec->length; i++) 433 for (i = 0; i < ib_conn->fmr.page_vec->length; i++)
434 iser_err("page_vec[%d] = 0x%llx\n", i, 434 iser_err("page_vec[%d] = 0x%llx\n", i,
435 (unsigned long long) ib_conn->fmr.page_vec->pages[i]); 435 (unsigned long long)ib_conn->fmr.page_vec->pages[i]);
436 } 436 }
437 if (err) 437 if (err)
438 return err; 438 return err;
@@ -440,77 +440,74 @@ int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
440 return 0; 440 return 0;
441} 441}
442 442
443static inline enum ib_t10_dif_type 443static inline void
444scsi2ib_prot_type(unsigned char prot_type) 444iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs,
445 struct ib_sig_domain *domain)
445{ 446{
446 switch (prot_type) { 447 domain->sig_type = IB_SIG_TYPE_T10_DIF;
447 case SCSI_PROT_DIF_TYPE0: 448 domain->sig.dif.pi_interval = sc->device->sector_size;
448 return IB_T10DIF_NONE; 449 domain->sig.dif.ref_tag = scsi_get_lba(sc) & 0xffffffff;
449 case SCSI_PROT_DIF_TYPE1: 450 /*
450 return IB_T10DIF_TYPE1; 451 * At the moment we hard code those, but in the future
451 case SCSI_PROT_DIF_TYPE2: 452 * we will take them from sc.
452 return IB_T10DIF_TYPE2; 453 */
453 case SCSI_PROT_DIF_TYPE3: 454 domain->sig.dif.apptag_check_mask = 0xffff;
454 return IB_T10DIF_TYPE3; 455 domain->sig.dif.app_escape = true;
455 default: 456 domain->sig.dif.ref_escape = true;
456 return IB_T10DIF_NONE; 457 if (scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE1 ||
457 } 458 scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE2)
458} 459 domain->sig.dif.ref_remap = true;
459 460};
460 461
461static int 462static int
462iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) 463iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs)
463{ 464{
464 unsigned char scsi_ptype = scsi_get_prot_type(sc);
465
466 sig_attrs->mem.sig_type = IB_SIG_TYPE_T10_DIF;
467 sig_attrs->wire.sig_type = IB_SIG_TYPE_T10_DIF;
468 sig_attrs->mem.sig.dif.pi_interval = sc->device->sector_size;
469 sig_attrs->wire.sig.dif.pi_interval = sc->device->sector_size;
470
471 switch (scsi_get_prot_op(sc)) { 465 switch (scsi_get_prot_op(sc)) {
472 case SCSI_PROT_WRITE_INSERT: 466 case SCSI_PROT_WRITE_INSERT:
473 case SCSI_PROT_READ_STRIP: 467 case SCSI_PROT_READ_STRIP:
474 sig_attrs->mem.sig.dif.type = IB_T10DIF_NONE; 468 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
475 sig_attrs->wire.sig.dif.type = scsi2ib_prot_type(scsi_ptype); 469 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
476 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 470 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
477 sig_attrs->wire.sig.dif.ref_tag = scsi_get_lba(sc) &
478 0xffffffff;
479 break; 471 break;
480 case SCSI_PROT_READ_INSERT: 472 case SCSI_PROT_READ_INSERT:
481 case SCSI_PROT_WRITE_STRIP: 473 case SCSI_PROT_WRITE_STRIP:
482 sig_attrs->mem.sig.dif.type = scsi2ib_prot_type(scsi_ptype); 474 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
483 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 475 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
484 sig_attrs->mem.sig.dif.ref_tag = scsi_get_lba(sc) & 476 /*
485 0xffffffff; 477 * At the moment we use this modparam to tell what is
486 sig_attrs->wire.sig.dif.type = IB_T10DIF_NONE; 478 * the memory bg_type, in the future we will take it
479 * from sc.
480 */
481 sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM :
482 IB_T10DIF_CRC;
487 break; 483 break;
488 case SCSI_PROT_READ_PASS: 484 case SCSI_PROT_READ_PASS:
489 case SCSI_PROT_WRITE_PASS: 485 case SCSI_PROT_WRITE_PASS:
490 sig_attrs->mem.sig.dif.type = scsi2ib_prot_type(scsi_ptype); 486 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire);
491 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC;
492 sig_attrs->mem.sig.dif.ref_tag = scsi_get_lba(sc) &
493 0xffffffff;
494 sig_attrs->wire.sig.dif.type = scsi2ib_prot_type(scsi_ptype);
495 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 487 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
496 sig_attrs->wire.sig.dif.ref_tag = scsi_get_lba(sc) & 488 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem);
497 0xffffffff; 489 /*
490 * At the moment we use this modparam to tell what is
491 * the memory bg_type, in the future we will take it
492 * from sc.
493 */
494 sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM :
495 IB_T10DIF_CRC;
498 break; 496 break;
499 default: 497 default:
500 iser_err("Unsupported PI operation %d\n", 498 iser_err("Unsupported PI operation %d\n",
501 scsi_get_prot_op(sc)); 499 scsi_get_prot_op(sc));
502 return -EINVAL; 500 return -EINVAL;
503 } 501 }
502
504 return 0; 503 return 0;
505} 504}
506 505
507
508static int 506static int
509iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 507iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask)
510{ 508{
511 switch (scsi_get_prot_type(sc)) { 509 switch (scsi_get_prot_type(sc)) {
512 case SCSI_PROT_DIF_TYPE0: 510 case SCSI_PROT_DIF_TYPE0:
513 *mask = 0x0;
514 break; 511 break;
515 case SCSI_PROT_DIF_TYPE1: 512 case SCSI_PROT_DIF_TYPE1:
516 case SCSI_PROT_DIF_TYPE2: 513 case SCSI_PROT_DIF_TYPE2:
@@ -533,7 +530,7 @@ iser_reg_sig_mr(struct iscsi_iser_task *iser_task,
533 struct fast_reg_descriptor *desc, struct ib_sge *data_sge, 530 struct fast_reg_descriptor *desc, struct ib_sge *data_sge,
534 struct ib_sge *prot_sge, struct ib_sge *sig_sge) 531 struct ib_sge *prot_sge, struct ib_sge *sig_sge)
535{ 532{
536 struct iser_conn *ib_conn = iser_task->ib_conn; 533 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
537 struct iser_pi_context *pi_ctx = desc->pi_ctx; 534 struct iser_pi_context *pi_ctx = desc->pi_ctx;
538 struct ib_send_wr sig_wr, inv_wr; 535 struct ib_send_wr sig_wr, inv_wr;
539 struct ib_send_wr *bad_wr, *wr = NULL; 536 struct ib_send_wr *bad_wr, *wr = NULL;
@@ -609,7 +606,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
609 struct ib_sge *sge) 606 struct ib_sge *sge)
610{ 607{
611 struct fast_reg_descriptor *desc = regd_buf->reg.mem_h; 608 struct fast_reg_descriptor *desc = regd_buf->reg.mem_h;
612 struct iser_conn *ib_conn = iser_task->ib_conn; 609 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
613 struct iser_device *device = ib_conn->device; 610 struct iser_device *device = ib_conn->device;
614 struct ib_device *ibdev = device->ib_device; 611 struct ib_device *ibdev = device->ib_device;
615 struct ib_mr *mr; 612 struct ib_mr *mr;
@@ -700,7 +697,7 @@ static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task,
700int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task, 697int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task,
701 enum iser_data_dir cmd_dir) 698 enum iser_data_dir cmd_dir)
702{ 699{
703 struct iser_conn *ib_conn = iser_task->ib_conn; 700 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn;
704 struct iser_device *device = ib_conn->device; 701 struct iser_device *device = ib_conn->device;
705 struct ib_device *ibdev = device->ib_device; 702 struct ib_device *ibdev = device->ib_device;
706 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 703 struct iser_data_buf *mem = &iser_task->data[cmd_dir];
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 3bfec4bbda52..67225bb82bb5 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -39,8 +39,12 @@
39#include "iscsi_iser.h" 39#include "iscsi_iser.h"
40 40
41#define ISCSI_ISER_MAX_CONN 8 41#define ISCSI_ISER_MAX_CONN 8
42#define ISER_MAX_RX_CQ_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN) 42#define ISER_MAX_RX_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN)
43#define ISER_MAX_TX_CQ_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN) 43#define ISER_MAX_TX_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN)
44#define ISER_MAX_CQ_LEN (ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \
45 ISCSI_ISER_MAX_CONN)
46
47static int iser_cq_poll_limit = 512;
44 48
45static void iser_cq_tasklet_fn(unsigned long data); 49static void iser_cq_tasklet_fn(unsigned long data);
46static void iser_cq_callback(struct ib_cq *cq, void *cq_context); 50static void iser_cq_callback(struct ib_cq *cq, void *cq_context);
@@ -71,7 +75,6 @@ static void iser_event_handler(struct ib_event_handler *handler,
71 */ 75 */
72static int iser_create_device_ib_res(struct iser_device *device) 76static int iser_create_device_ib_res(struct iser_device *device)
73{ 77{
74 struct iser_cq_desc *cq_desc;
75 struct ib_device_attr *dev_attr = &device->dev_attr; 78 struct ib_device_attr *dev_attr = &device->dev_attr;
76 int ret, i; 79 int ret, i;
77 80
@@ -101,51 +104,35 @@ static int iser_create_device_ib_res(struct iser_device *device)
101 return -1; 104 return -1;
102 } 105 }
103 106
104 device->cqs_used = min(ISER_MAX_CQ, device->ib_device->num_comp_vectors); 107 device->comps_used = min(ISER_MAX_CQ,
108 device->ib_device->num_comp_vectors);
105 iser_info("using %d CQs, device %s supports %d vectors\n", 109 iser_info("using %d CQs, device %s supports %d vectors\n",
106 device->cqs_used, device->ib_device->name, 110 device->comps_used, device->ib_device->name,
107 device->ib_device->num_comp_vectors); 111 device->ib_device->num_comp_vectors);
108 112
109 device->cq_desc = kmalloc(sizeof(struct iser_cq_desc) * device->cqs_used,
110 GFP_KERNEL);
111 if (device->cq_desc == NULL)
112 goto cq_desc_err;
113 cq_desc = device->cq_desc;
114
115 device->pd = ib_alloc_pd(device->ib_device); 113 device->pd = ib_alloc_pd(device->ib_device);
116 if (IS_ERR(device->pd)) 114 if (IS_ERR(device->pd))
117 goto pd_err; 115 goto pd_err;
118 116
119 for (i = 0; i < device->cqs_used; i++) { 117 for (i = 0; i < device->comps_used; i++) {
120 cq_desc[i].device = device; 118 struct iser_comp *comp = &device->comps[i];
121 cq_desc[i].cq_index = i; 119
122 120 comp->device = device;
123 device->rx_cq[i] = ib_create_cq(device->ib_device, 121 comp->cq = ib_create_cq(device->ib_device,
124 iser_cq_callback, 122 iser_cq_callback,
125 iser_cq_event_callback, 123 iser_cq_event_callback,
126 (void *)&cq_desc[i], 124 (void *)comp,
127 ISER_MAX_RX_CQ_LEN, i); 125 ISER_MAX_CQ_LEN, i);
128 if (IS_ERR(device->rx_cq[i])) { 126 if (IS_ERR(comp->cq)) {
129 device->rx_cq[i] = NULL; 127 comp->cq = NULL;
130 goto cq_err; 128 goto cq_err;
131 } 129 }
132 130
133 device->tx_cq[i] = ib_create_cq(device->ib_device, 131 if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP))
134 NULL, iser_cq_event_callback,
135 (void *)&cq_desc[i],
136 ISER_MAX_TX_CQ_LEN, i);
137
138 if (IS_ERR(device->tx_cq[i])) {
139 device->tx_cq[i] = NULL;
140 goto cq_err; 132 goto cq_err;
141 }
142 133
143 if (ib_req_notify_cq(device->rx_cq[i], IB_CQ_NEXT_COMP)) 134 tasklet_init(&comp->tasklet, iser_cq_tasklet_fn,
144 goto cq_err; 135 (unsigned long)comp);
145
146 tasklet_init(&device->cq_tasklet[i],
147 iser_cq_tasklet_fn,
148 (unsigned long)&cq_desc[i]);
149 } 136 }
150 137
151 device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE | 138 device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE |
@@ -164,19 +151,17 @@ static int iser_create_device_ib_res(struct iser_device *device)
164handler_err: 151handler_err:
165 ib_dereg_mr(device->mr); 152 ib_dereg_mr(device->mr);
166dma_mr_err: 153dma_mr_err:
167 for (i = 0; i < device->cqs_used; i++) 154 for (i = 0; i < device->comps_used; i++)
168 tasklet_kill(&device->cq_tasklet[i]); 155 tasklet_kill(&device->comps[i].tasklet);
169cq_err: 156cq_err:
170 for (i = 0; i < device->cqs_used; i++) { 157 for (i = 0; i < device->comps_used; i++) {
171 if (device->tx_cq[i]) 158 struct iser_comp *comp = &device->comps[i];
172 ib_destroy_cq(device->tx_cq[i]); 159
173 if (device->rx_cq[i]) 160 if (comp->cq)
174 ib_destroy_cq(device->rx_cq[i]); 161 ib_destroy_cq(comp->cq);
175 } 162 }
176 ib_dealloc_pd(device->pd); 163 ib_dealloc_pd(device->pd);
177pd_err: 164pd_err:
178 kfree(device->cq_desc);
179cq_desc_err:
180 iser_err("failed to allocate an IB resource\n"); 165 iser_err("failed to allocate an IB resource\n");
181 return -1; 166 return -1;
182} 167}
@@ -190,20 +175,18 @@ static void iser_free_device_ib_res(struct iser_device *device)
190 int i; 175 int i;
191 BUG_ON(device->mr == NULL); 176 BUG_ON(device->mr == NULL);
192 177
193 for (i = 0; i < device->cqs_used; i++) { 178 for (i = 0; i < device->comps_used; i++) {
194 tasklet_kill(&device->cq_tasklet[i]); 179 struct iser_comp *comp = &device->comps[i];
195 (void)ib_destroy_cq(device->tx_cq[i]); 180
196 (void)ib_destroy_cq(device->rx_cq[i]); 181 tasklet_kill(&comp->tasklet);
197 device->tx_cq[i] = NULL; 182 ib_destroy_cq(comp->cq);
198 device->rx_cq[i] = NULL; 183 comp->cq = NULL;
199 } 184 }
200 185
201 (void)ib_unregister_event_handler(&device->event_handler); 186 (void)ib_unregister_event_handler(&device->event_handler);
202 (void)ib_dereg_mr(device->mr); 187 (void)ib_dereg_mr(device->mr);
203 (void)ib_dealloc_pd(device->pd); 188 (void)ib_dealloc_pd(device->pd);
204 189
205 kfree(device->cq_desc);
206
207 device->mr = NULL; 190 device->mr = NULL;
208 device->pd = NULL; 191 device->pd = NULL;
209} 192}
@@ -213,7 +196,7 @@ static void iser_free_device_ib_res(struct iser_device *device)
213 * 196 *
214 * returns 0 on success, or errno code on failure 197 * returns 0 on success, or errno code on failure
215 */ 198 */
216int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max) 199int iser_create_fmr_pool(struct ib_conn *ib_conn, unsigned cmds_max)
217{ 200{
218 struct iser_device *device = ib_conn->device; 201 struct iser_device *device = ib_conn->device;
219 struct ib_fmr_pool_param params; 202 struct ib_fmr_pool_param params;
@@ -263,7 +246,7 @@ int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max)
263/** 246/**
264 * iser_free_fmr_pool - releases the FMR pool and page vec 247 * iser_free_fmr_pool - releases the FMR pool and page vec
265 */ 248 */
266void iser_free_fmr_pool(struct iser_conn *ib_conn) 249void iser_free_fmr_pool(struct ib_conn *ib_conn)
267{ 250{
268 iser_info("freeing conn %p fmr pool %p\n", 251 iser_info("freeing conn %p fmr pool %p\n",
269 ib_conn, ib_conn->fmr.pool); 252 ib_conn, ib_conn->fmr.pool);
@@ -367,10 +350,10 @@ fast_reg_mr_failure:
367 * for fast registration work requests. 350 * for fast registration work requests.
368 * returns 0 on success, or errno code on failure 351 * returns 0 on success, or errno code on failure
369 */ 352 */
370int iser_create_fastreg_pool(struct iser_conn *ib_conn, unsigned cmds_max) 353int iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max)
371{ 354{
372 struct iser_device *device = ib_conn->device; 355 struct iser_device *device = ib_conn->device;
373 struct fast_reg_descriptor *desc; 356 struct fast_reg_descriptor *desc;
374 int i, ret; 357 int i, ret;
375 358
376 INIT_LIST_HEAD(&ib_conn->fastreg.pool); 359 INIT_LIST_HEAD(&ib_conn->fastreg.pool);
@@ -406,7 +389,7 @@ err:
406/** 389/**
407 * iser_free_fastreg_pool - releases the pool of fast_reg descriptors 390 * iser_free_fastreg_pool - releases the pool of fast_reg descriptors
408 */ 391 */
409void iser_free_fastreg_pool(struct iser_conn *ib_conn) 392void iser_free_fastreg_pool(struct ib_conn *ib_conn)
410{ 393{
411 struct fast_reg_descriptor *desc, *tmp; 394 struct fast_reg_descriptor *desc, *tmp;
412 int i = 0; 395 int i = 0;
@@ -440,7 +423,7 @@ void iser_free_fastreg_pool(struct iser_conn *ib_conn)
440 * 423 *
441 * returns 0 on success, -1 on failure 424 * returns 0 on success, -1 on failure
442 */ 425 */
443static int iser_create_ib_conn_res(struct iser_conn *ib_conn) 426static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
444{ 427{
445 struct iser_device *device; 428 struct iser_device *device;
446 struct ib_qp_init_attr init_attr; 429 struct ib_qp_init_attr init_attr;
@@ -455,28 +438,30 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
455 438
456 mutex_lock(&ig.connlist_mutex); 439 mutex_lock(&ig.connlist_mutex);
457 /* select the CQ with the minimal number of usages */ 440 /* select the CQ with the minimal number of usages */
458 for (index = 0; index < device->cqs_used; index++) 441 for (index = 0; index < device->comps_used; index++) {
459 if (device->cq_active_qps[index] < 442 if (device->comps[index].active_qps <
460 device->cq_active_qps[min_index]) 443 device->comps[min_index].active_qps)
461 min_index = index; 444 min_index = index;
462 device->cq_active_qps[min_index]++; 445 }
446 ib_conn->comp = &device->comps[min_index];
447 ib_conn->comp->active_qps++;
463 mutex_unlock(&ig.connlist_mutex); 448 mutex_unlock(&ig.connlist_mutex);
464 iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn); 449 iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn);
465 450
466 init_attr.event_handler = iser_qp_event_callback; 451 init_attr.event_handler = iser_qp_event_callback;
467 init_attr.qp_context = (void *)ib_conn; 452 init_attr.qp_context = (void *)ib_conn;
468 init_attr.send_cq = device->tx_cq[min_index]; 453 init_attr.send_cq = ib_conn->comp->cq;
469 init_attr.recv_cq = device->rx_cq[min_index]; 454 init_attr.recv_cq = ib_conn->comp->cq;
470 init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS; 455 init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;
471 init_attr.cap.max_send_sge = 2; 456 init_attr.cap.max_send_sge = 2;
472 init_attr.cap.max_recv_sge = 1; 457 init_attr.cap.max_recv_sge = 1;
473 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 458 init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
474 init_attr.qp_type = IB_QPT_RC; 459 init_attr.qp_type = IB_QPT_RC;
475 if (ib_conn->pi_support) { 460 if (ib_conn->pi_support) {
476 init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS; 461 init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
477 init_attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 462 init_attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
478 } else { 463 } else {
479 init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS; 464 init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS + 1;
480 } 465 }
481 466
482 ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr); 467 ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);
@@ -495,30 +480,6 @@ out_err:
495} 480}
496 481
497/** 482/**
498 * releases the QP object
499 */
500static void iser_free_ib_conn_res(struct iser_conn *ib_conn)
501{
502 int cq_index;
503 BUG_ON(ib_conn == NULL);
504
505 iser_info("freeing conn %p cma_id %p qp %p\n",
506 ib_conn, ib_conn->cma_id,
507 ib_conn->qp);
508
509 /* qp is created only once both addr & route are resolved */
510
511 if (ib_conn->qp != NULL) {
512 cq_index = ((struct iser_cq_desc *)ib_conn->qp->recv_cq->cq_context)->cq_index;
513 ib_conn->device->cq_active_qps[cq_index]--;
514
515 rdma_destroy_qp(ib_conn->cma_id);
516 }
517
518 ib_conn->qp = NULL;
519}
520
521/**
522 * based on the resolved device node GUID see if there already allocated 483 * based on the resolved device node GUID see if there already allocated
523 * device for this device. If there's no such, create one. 484 * device for this device. If there's no such, create one.
524 */ 485 */
@@ -572,88 +533,142 @@ static void iser_device_try_release(struct iser_device *device)
572/** 533/**
573 * Called with state mutex held 534 * Called with state mutex held
574 **/ 535 **/
575static int iser_conn_state_comp_exch(struct iser_conn *ib_conn, 536static int iser_conn_state_comp_exch(struct iser_conn *iser_conn,
576 enum iser_ib_conn_state comp, 537 enum iser_conn_state comp,
577 enum iser_ib_conn_state exch) 538 enum iser_conn_state exch)
578{ 539{
579 int ret; 540 int ret;
580 541
581 if ((ret = (ib_conn->state == comp))) 542 ret = (iser_conn->state == comp);
582 ib_conn->state = exch; 543 if (ret)
544 iser_conn->state = exch;
545
583 return ret; 546 return ret;
584} 547}
585 548
586void iser_release_work(struct work_struct *work) 549void iser_release_work(struct work_struct *work)
587{ 550{
588 struct iser_conn *ib_conn; 551 struct iser_conn *iser_conn;
589 int rc;
590 552
591 ib_conn = container_of(work, struct iser_conn, release_work); 553 iser_conn = container_of(work, struct iser_conn, release_work);
592 554
593 /* wait for .conn_stop callback */ 555 /* Wait for conn_stop to complete */
594 rc = wait_for_completion_timeout(&ib_conn->stop_completion, 30 * HZ); 556 wait_for_completion(&iser_conn->stop_completion);
595 WARN_ON(rc == 0); 557 /* Wait for IB resouces cleanup to complete */
558 wait_for_completion(&iser_conn->ib_completion);
596 559
597 /* wait for the qp`s post send and post receive buffers to empty */ 560 mutex_lock(&iser_conn->state_mutex);
598 rc = wait_for_completion_timeout(&ib_conn->flush_completion, 30 * HZ); 561 iser_conn->state = ISER_CONN_DOWN;
599 WARN_ON(rc == 0); 562 mutex_unlock(&iser_conn->state_mutex);
600 563
601 ib_conn->state = ISER_CONN_DOWN; 564 iser_conn_release(iser_conn);
565}
566
567/**
568 * iser_free_ib_conn_res - release IB related resources
569 * @iser_conn: iser connection struct
570 * @destroy_device: indicator if we need to try to release
571 * the iser device (only iscsi shutdown and DEVICE_REMOVAL
572 * will use this.
573 *
574 * This routine is called with the iser state mutex held
575 * so the cm_id removal is out of here. It is Safe to
576 * be invoked multiple times.
577 */
578static void iser_free_ib_conn_res(struct iser_conn *iser_conn,
579 bool destroy_device)
580{
581 struct ib_conn *ib_conn = &iser_conn->ib_conn;
582 struct iser_device *device = ib_conn->device;
602 583
603 mutex_lock(&ib_conn->state_mutex); 584 iser_info("freeing conn %p cma_id %p qp %p\n",
604 ib_conn->state = ISER_CONN_DOWN; 585 iser_conn, ib_conn->cma_id, ib_conn->qp);
605 mutex_unlock(&ib_conn->state_mutex); 586
587 iser_free_rx_descriptors(iser_conn);
606 588
607 iser_conn_release(ib_conn); 589 if (ib_conn->qp != NULL) {
590 ib_conn->comp->active_qps--;
591 rdma_destroy_qp(ib_conn->cma_id);
592 ib_conn->qp = NULL;
593 }
594
595 if (destroy_device && device != NULL) {
596 iser_device_try_release(device);
597 ib_conn->device = NULL;
598 }
608} 599}
609 600
610/** 601/**
611 * Frees all conn objects and deallocs conn descriptor 602 * Frees all conn objects and deallocs conn descriptor
612 */ 603 */
613void iser_conn_release(struct iser_conn *ib_conn) 604void iser_conn_release(struct iser_conn *iser_conn)
614{ 605{
615 struct iser_device *device = ib_conn->device; 606 struct ib_conn *ib_conn = &iser_conn->ib_conn;
616 607
617 mutex_lock(&ig.connlist_mutex); 608 mutex_lock(&ig.connlist_mutex);
618 list_del(&ib_conn->conn_list); 609 list_del(&iser_conn->conn_list);
619 mutex_unlock(&ig.connlist_mutex); 610 mutex_unlock(&ig.connlist_mutex);
620 611
621 mutex_lock(&ib_conn->state_mutex); 612 mutex_lock(&iser_conn->state_mutex);
622 BUG_ON(ib_conn->state != ISER_CONN_DOWN); 613 if (iser_conn->state != ISER_CONN_DOWN)
623 614 iser_warn("iser conn %p state %d, expected state down.\n",
624 iser_free_rx_descriptors(ib_conn); 615 iser_conn, iser_conn->state);
625 iser_free_ib_conn_res(ib_conn); 616 /*
626 ib_conn->device = NULL; 617 * In case we never got to bind stage, we still need to
627 /* on EVENT_ADDR_ERROR there's no device yet for this conn */ 618 * release IB resources (which is safe to call more than once).
628 if (device != NULL) 619 */
629 iser_device_try_release(device); 620 iser_free_ib_conn_res(iser_conn, true);
630 mutex_unlock(&ib_conn->state_mutex); 621 mutex_unlock(&iser_conn->state_mutex);
631 622
632 /* if cma handler context, the caller actually destroy the id */
633 if (ib_conn->cma_id != NULL) { 623 if (ib_conn->cma_id != NULL) {
634 rdma_destroy_id(ib_conn->cma_id); 624 rdma_destroy_id(ib_conn->cma_id);
635 ib_conn->cma_id = NULL; 625 ib_conn->cma_id = NULL;
636 } 626 }
637 kfree(ib_conn); 627
628 kfree(iser_conn);
638} 629}
639 630
640/** 631/**
641 * triggers start of the disconnect procedures and wait for them to be done 632 * triggers start of the disconnect procedures and wait for them to be done
633 * Called with state mutex held
642 */ 634 */
643void iser_conn_terminate(struct iser_conn *ib_conn) 635int iser_conn_terminate(struct iser_conn *iser_conn)
644{ 636{
637 struct ib_conn *ib_conn = &iser_conn->ib_conn;
638 struct ib_send_wr *bad_wr;
645 int err = 0; 639 int err = 0;
646 640
647 /* change the ib conn state only if the conn is UP, however always call 641 /* terminate the iser conn only if the conn state is UP */
648 * rdma_disconnect since this is the only way to cause the CMA to change 642 if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP,
649 * the QP state to ERROR 643 ISER_CONN_TERMINATING))
644 return 0;
645
646 iser_info("iser_conn %p state %d\n", iser_conn, iser_conn->state);
647
648 /* suspend queuing of new iscsi commands */
649 if (iser_conn->iscsi_conn)
650 iscsi_suspend_queue(iser_conn->iscsi_conn);
651
652 /*
653 * In case we didn't already clean up the cma_id (peer initiated
654 * a disconnection), we need to Cause the CMA to change the QP
655 * state to ERROR.
650 */ 656 */
657 if (ib_conn->cma_id) {
658 err = rdma_disconnect(ib_conn->cma_id);
659 if (err)
660 iser_err("Failed to disconnect, conn: 0x%p err %d\n",
661 iser_conn, err);
662
663 /* post an indication that all flush errors were consumed */
664 err = ib_post_send(ib_conn->qp, &ib_conn->beacon, &bad_wr);
665 if (err)
666 iser_err("conn %p failed to post beacon", ib_conn);
667
668 wait_for_completion(&ib_conn->flush_comp);
669 }
651 670
652 iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, ISER_CONN_TERMINATING); 671 return 1;
653 err = rdma_disconnect(ib_conn->cma_id);
654 if (err)
655 iser_err("Failed to disconnect, conn: 0x%p err %d\n",
656 ib_conn,err);
657} 672}
658 673
659/** 674/**
@@ -661,10 +676,10 @@ void iser_conn_terminate(struct iser_conn *ib_conn)
661 **/ 676 **/
662static void iser_connect_error(struct rdma_cm_id *cma_id) 677static void iser_connect_error(struct rdma_cm_id *cma_id)
663{ 678{
664 struct iser_conn *ib_conn; 679 struct iser_conn *iser_conn;
665 680
666 ib_conn = (struct iser_conn *)cma_id->context; 681 iser_conn = (struct iser_conn *)cma_id->context;
667 ib_conn->state = ISER_CONN_DOWN; 682 iser_conn->state = ISER_CONN_DOWN;
668} 683}
669 684
670/** 685/**
@@ -673,14 +688,16 @@ static void iser_connect_error(struct rdma_cm_id *cma_id)
673static void iser_addr_handler(struct rdma_cm_id *cma_id) 688static void iser_addr_handler(struct rdma_cm_id *cma_id)
674{ 689{
675 struct iser_device *device; 690 struct iser_device *device;
676 struct iser_conn *ib_conn; 691 struct iser_conn *iser_conn;
692 struct ib_conn *ib_conn;
677 int ret; 693 int ret;
678 694
679 ib_conn = (struct iser_conn *)cma_id->context; 695 iser_conn = (struct iser_conn *)cma_id->context;
680 if (ib_conn->state != ISER_CONN_PENDING) 696 if (iser_conn->state != ISER_CONN_PENDING)
681 /* bailout */ 697 /* bailout */
682 return; 698 return;
683 699
700 ib_conn = &iser_conn->ib_conn;
684 device = iser_device_find_by_ib_device(cma_id); 701 device = iser_device_find_by_ib_device(cma_id);
685 if (!device) { 702 if (!device) {
686 iser_err("device lookup/creation failed\n"); 703 iser_err("device lookup/creation failed\n");
@@ -719,14 +736,15 @@ static void iser_route_handler(struct rdma_cm_id *cma_id)
719 struct rdma_conn_param conn_param; 736 struct rdma_conn_param conn_param;
720 int ret; 737 int ret;
721 struct iser_cm_hdr req_hdr; 738 struct iser_cm_hdr req_hdr;
722 struct iser_conn *ib_conn = (struct iser_conn *)cma_id->context; 739 struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
740 struct ib_conn *ib_conn = &iser_conn->ib_conn;
723 struct iser_device *device = ib_conn->device; 741 struct iser_device *device = ib_conn->device;
724 742
725 if (ib_conn->state != ISER_CONN_PENDING) 743 if (iser_conn->state != ISER_CONN_PENDING)
726 /* bailout */ 744 /* bailout */
727 return; 745 return;
728 746
729 ret = iser_create_ib_conn_res((struct iser_conn *)cma_id->context); 747 ret = iser_create_ib_conn_res(ib_conn);
730 if (ret) 748 if (ret)
731 goto failure; 749 goto failure;
732 750
@@ -755,57 +773,60 @@ failure:
755 773
756static void iser_connected_handler(struct rdma_cm_id *cma_id) 774static void iser_connected_handler(struct rdma_cm_id *cma_id)
757{ 775{
758 struct iser_conn *ib_conn; 776 struct iser_conn *iser_conn;
759 struct ib_qp_attr attr; 777 struct ib_qp_attr attr;
760 struct ib_qp_init_attr init_attr; 778 struct ib_qp_init_attr init_attr;
761 779
762 ib_conn = (struct iser_conn *)cma_id->context; 780 iser_conn = (struct iser_conn *)cma_id->context;
763 if (ib_conn->state != ISER_CONN_PENDING) 781 if (iser_conn->state != ISER_CONN_PENDING)
764 /* bailout */ 782 /* bailout */
765 return; 783 return;
766 784
767 (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr); 785 (void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr);
768 iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num); 786 iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num);
769 787
770 ib_conn->state = ISER_CONN_UP; 788 iser_conn->state = ISER_CONN_UP;
771 complete(&ib_conn->up_completion); 789 complete(&iser_conn->up_completion);
772} 790}
773 791
774static void iser_disconnected_handler(struct rdma_cm_id *cma_id) 792static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
775{ 793{
776 struct iser_conn *ib_conn; 794 struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
777
778 ib_conn = (struct iser_conn *)cma_id->context;
779 795
780 /* getting here when the state is UP means that the conn is being * 796 if (iser_conn_terminate(iser_conn)) {
781 * terminated asynchronously from the iSCSI layer's perspective. */ 797 if (iser_conn->iscsi_conn)
782 if (iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, 798 iscsi_conn_failure(iser_conn->iscsi_conn,
783 ISER_CONN_TERMINATING)){ 799 ISCSI_ERR_CONN_FAILED);
784 if (ib_conn->iscsi_conn)
785 iscsi_conn_failure(ib_conn->iscsi_conn, ISCSI_ERR_CONN_FAILED);
786 else 800 else
787 iser_err("iscsi_iser connection isn't bound\n"); 801 iser_err("iscsi_iser connection isn't bound\n");
788 } 802 }
803}
804
805static void iser_cleanup_handler(struct rdma_cm_id *cma_id,
806 bool destroy_device)
807{
808 struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
789 809
790 /* Complete the termination process if no posts are pending. This code 810 /*
791 * block also exists in iser_handle_comp_error(), but it is needed here 811 * We are not guaranteed that we visited disconnected_handler
792 * for cases of no flushes at all, e.g. discovery over rdma. 812 * by now, call it here to be safe that we handle CM drep
813 * and flush errors.
793 */ 814 */
794 if (ib_conn->post_recv_buf_count == 0 && 815 iser_disconnected_handler(cma_id);
795 (atomic_read(&ib_conn->post_send_buf_count) == 0)) { 816 iser_free_ib_conn_res(iser_conn, destroy_device);
796 complete(&ib_conn->flush_completion); 817 complete(&iser_conn->ib_completion);
797 } 818};
798}
799 819
800static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 820static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
801{ 821{
802 struct iser_conn *ib_conn; 822 struct iser_conn *iser_conn;
823 int ret = 0;
803 824
804 ib_conn = (struct iser_conn *)cma_id->context; 825 iser_conn = (struct iser_conn *)cma_id->context;
805 iser_info("event %d status %d conn %p id %p\n", 826 iser_info("event %d status %d conn %p id %p\n",
806 event->event, event->status, cma_id->context, cma_id); 827 event->event, event->status, cma_id->context, cma_id);
807 828
808 mutex_lock(&ib_conn->state_mutex); 829 mutex_lock(&iser_conn->state_mutex);
809 switch (event->event) { 830 switch (event->event) {
810 case RDMA_CM_EVENT_ADDR_RESOLVED: 831 case RDMA_CM_EVENT_ADDR_RESOLVED:
811 iser_addr_handler(cma_id); 832 iser_addr_handler(cma_id);
@@ -824,57 +845,73 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
824 iser_connect_error(cma_id); 845 iser_connect_error(cma_id);
825 break; 846 break;
826 case RDMA_CM_EVENT_DISCONNECTED: 847 case RDMA_CM_EVENT_DISCONNECTED:
827 case RDMA_CM_EVENT_DEVICE_REMOVAL:
828 case RDMA_CM_EVENT_ADDR_CHANGE: 848 case RDMA_CM_EVENT_ADDR_CHANGE:
829 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
830 iser_disconnected_handler(cma_id); 849 iser_disconnected_handler(cma_id);
831 break; 850 break;
851 case RDMA_CM_EVENT_DEVICE_REMOVAL:
852 /*
853 * we *must* destroy the device as we cannot rely
854 * on iscsid to be around to initiate error handling.
855 * also implicitly destroy the cma_id.
856 */
857 iser_cleanup_handler(cma_id, true);
858 iser_conn->ib_conn.cma_id = NULL;
859 ret = 1;
860 break;
861 case RDMA_CM_EVENT_TIMEWAIT_EXIT:
862 iser_cleanup_handler(cma_id, false);
863 break;
832 default: 864 default:
833 iser_err("Unexpected RDMA CM event (%d)\n", event->event); 865 iser_err("Unexpected RDMA CM event (%d)\n", event->event);
834 break; 866 break;
835 } 867 }
836 mutex_unlock(&ib_conn->state_mutex); 868 mutex_unlock(&iser_conn->state_mutex);
837 return 0; 869
870 return ret;
838} 871}
839 872
840void iser_conn_init(struct iser_conn *ib_conn) 873void iser_conn_init(struct iser_conn *iser_conn)
841{ 874{
842 ib_conn->state = ISER_CONN_INIT; 875 iser_conn->state = ISER_CONN_INIT;
843 ib_conn->post_recv_buf_count = 0; 876 iser_conn->ib_conn.post_recv_buf_count = 0;
844 atomic_set(&ib_conn->post_send_buf_count, 0); 877 init_completion(&iser_conn->ib_conn.flush_comp);
845 init_completion(&ib_conn->stop_completion); 878 init_completion(&iser_conn->stop_completion);
846 init_completion(&ib_conn->flush_completion); 879 init_completion(&iser_conn->ib_completion);
847 init_completion(&ib_conn->up_completion); 880 init_completion(&iser_conn->up_completion);
848 INIT_LIST_HEAD(&ib_conn->conn_list); 881 INIT_LIST_HEAD(&iser_conn->conn_list);
849 spin_lock_init(&ib_conn->lock); 882 spin_lock_init(&iser_conn->ib_conn.lock);
850 mutex_init(&ib_conn->state_mutex); 883 mutex_init(&iser_conn->state_mutex);
851} 884}
852 885
853 /** 886 /**
854 * starts the process of connecting to the target 887 * starts the process of connecting to the target
855 * sleeps until the connection is established or rejected 888 * sleeps until the connection is established or rejected
856 */ 889 */
857int iser_connect(struct iser_conn *ib_conn, 890int iser_connect(struct iser_conn *iser_conn,
858 struct sockaddr *src_addr, 891 struct sockaddr *src_addr,
859 struct sockaddr *dst_addr, 892 struct sockaddr *dst_addr,
860 int non_blocking) 893 int non_blocking)
861{ 894{
895 struct ib_conn *ib_conn = &iser_conn->ib_conn;
862 int err = 0; 896 int err = 0;
863 897
864 mutex_lock(&ib_conn->state_mutex); 898 mutex_lock(&iser_conn->state_mutex);
865 899
866 sprintf(ib_conn->name, "%pISp", dst_addr); 900 sprintf(iser_conn->name, "%pISp", dst_addr);
867 901
868 iser_info("connecting to: %s\n", ib_conn->name); 902 iser_info("connecting to: %s\n", iser_conn->name);
869 903
870 /* the device is known only --after-- address resolution */ 904 /* the device is known only --after-- address resolution */
871 ib_conn->device = NULL; 905 ib_conn->device = NULL;
872 906
873 ib_conn->state = ISER_CONN_PENDING; 907 iser_conn->state = ISER_CONN_PENDING;
908
909 ib_conn->beacon.wr_id = ISER_BEACON_WRID;
910 ib_conn->beacon.opcode = IB_WR_SEND;
874 911
875 ib_conn->cma_id = rdma_create_id(iser_cma_handler, 912 ib_conn->cma_id = rdma_create_id(iser_cma_handler,
876 (void *)ib_conn, 913 (void *)iser_conn,
877 RDMA_PS_TCP, IB_QPT_RC); 914 RDMA_PS_TCP, IB_QPT_RC);
878 if (IS_ERR(ib_conn->cma_id)) { 915 if (IS_ERR(ib_conn->cma_id)) {
879 err = PTR_ERR(ib_conn->cma_id); 916 err = PTR_ERR(ib_conn->cma_id);
880 iser_err("rdma_create_id failed: %d\n", err); 917 iser_err("rdma_create_id failed: %d\n", err);
@@ -888,27 +925,27 @@ int iser_connect(struct iser_conn *ib_conn,
888 } 925 }
889 926
890 if (!non_blocking) { 927 if (!non_blocking) {
891 wait_for_completion_interruptible(&ib_conn->up_completion); 928 wait_for_completion_interruptible(&iser_conn->up_completion);
892 929
893 if (ib_conn->state != ISER_CONN_UP) { 930 if (iser_conn->state != ISER_CONN_UP) {
894 err = -EIO; 931 err = -EIO;
895 goto connect_failure; 932 goto connect_failure;
896 } 933 }
897 } 934 }
898 mutex_unlock(&ib_conn->state_mutex); 935 mutex_unlock(&iser_conn->state_mutex);
899 936
900 mutex_lock(&ig.connlist_mutex); 937 mutex_lock(&ig.connlist_mutex);
901 list_add(&ib_conn->conn_list, &ig.connlist); 938 list_add(&iser_conn->conn_list, &ig.connlist);
902 mutex_unlock(&ig.connlist_mutex); 939 mutex_unlock(&ig.connlist_mutex);
903 return 0; 940 return 0;
904 941
905id_failure: 942id_failure:
906 ib_conn->cma_id = NULL; 943 ib_conn->cma_id = NULL;
907addr_failure: 944addr_failure:
908 ib_conn->state = ISER_CONN_DOWN; 945 iser_conn->state = ISER_CONN_DOWN;
909connect_failure: 946connect_failure:
910 mutex_unlock(&ib_conn->state_mutex); 947 mutex_unlock(&iser_conn->state_mutex);
911 iser_conn_release(ib_conn); 948 iser_conn_release(iser_conn);
912 return err; 949 return err;
913} 950}
914 951
@@ -917,7 +954,7 @@ connect_failure:
917 * 954 *
918 * returns: 0 on success, errno code on failure 955 * returns: 0 on success, errno code on failure
919 */ 956 */
920int iser_reg_page_vec(struct iser_conn *ib_conn, 957int iser_reg_page_vec(struct ib_conn *ib_conn,
921 struct iser_page_vec *page_vec, 958 struct iser_page_vec *page_vec,
922 struct iser_mem_reg *mem_reg) 959 struct iser_mem_reg *mem_reg)
923{ 960{
@@ -987,7 +1024,8 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
987 enum iser_data_dir cmd_dir) 1024 enum iser_data_dir cmd_dir)
988{ 1025{
989 struct iser_mem_reg *reg = &iser_task->rdma_regd[cmd_dir].reg; 1026 struct iser_mem_reg *reg = &iser_task->rdma_regd[cmd_dir].reg;
990 struct iser_conn *ib_conn = iser_task->ib_conn; 1027 struct iser_conn *iser_conn = iser_task->iser_conn;
1028 struct ib_conn *ib_conn = &iser_conn->ib_conn;
991 struct fast_reg_descriptor *desc = reg->mem_h; 1029 struct fast_reg_descriptor *desc = reg->mem_h;
992 1030
993 if (!reg->is_mr) 1031 if (!reg->is_mr)
@@ -1000,17 +1038,18 @@ void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
1000 spin_unlock_bh(&ib_conn->lock); 1038 spin_unlock_bh(&ib_conn->lock);
1001} 1039}
1002 1040
1003int iser_post_recvl(struct iser_conn *ib_conn) 1041int iser_post_recvl(struct iser_conn *iser_conn)
1004{ 1042{
1005 struct ib_recv_wr rx_wr, *rx_wr_failed; 1043 struct ib_recv_wr rx_wr, *rx_wr_failed;
1044 struct ib_conn *ib_conn = &iser_conn->ib_conn;
1006 struct ib_sge sge; 1045 struct ib_sge sge;
1007 int ib_ret; 1046 int ib_ret;
1008 1047
1009 sge.addr = ib_conn->login_resp_dma; 1048 sge.addr = iser_conn->login_resp_dma;
1010 sge.length = ISER_RX_LOGIN_SIZE; 1049 sge.length = ISER_RX_LOGIN_SIZE;
1011 sge.lkey = ib_conn->device->mr->lkey; 1050 sge.lkey = ib_conn->device->mr->lkey;
1012 1051
1013 rx_wr.wr_id = (unsigned long)ib_conn->login_resp_buf; 1052 rx_wr.wr_id = (unsigned long)iser_conn->login_resp_buf;
1014 rx_wr.sg_list = &sge; 1053 rx_wr.sg_list = &sge;
1015 rx_wr.num_sge = 1; 1054 rx_wr.num_sge = 1;
1016 rx_wr.next = NULL; 1055 rx_wr.next = NULL;
@@ -1024,20 +1063,21 @@ int iser_post_recvl(struct iser_conn *ib_conn)
1024 return ib_ret; 1063 return ib_ret;
1025} 1064}
1026 1065
1027int iser_post_recvm(struct iser_conn *ib_conn, int count) 1066int iser_post_recvm(struct iser_conn *iser_conn, int count)
1028{ 1067{
1029 struct ib_recv_wr *rx_wr, *rx_wr_failed; 1068 struct ib_recv_wr *rx_wr, *rx_wr_failed;
1030 int i, ib_ret; 1069 int i, ib_ret;
1031 unsigned int my_rx_head = ib_conn->rx_desc_head; 1070 struct ib_conn *ib_conn = &iser_conn->ib_conn;
1071 unsigned int my_rx_head = iser_conn->rx_desc_head;
1032 struct iser_rx_desc *rx_desc; 1072 struct iser_rx_desc *rx_desc;
1033 1073
1034 for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 1074 for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
1035 rx_desc = &ib_conn->rx_descs[my_rx_head]; 1075 rx_desc = &iser_conn->rx_descs[my_rx_head];
1036 rx_wr->wr_id = (unsigned long)rx_desc; 1076 rx_wr->wr_id = (unsigned long)rx_desc;
1037 rx_wr->sg_list = &rx_desc->rx_sg; 1077 rx_wr->sg_list = &rx_desc->rx_sg;
1038 rx_wr->num_sge = 1; 1078 rx_wr->num_sge = 1;
1039 rx_wr->next = rx_wr + 1; 1079 rx_wr->next = rx_wr + 1;
1040 my_rx_head = (my_rx_head + 1) & ib_conn->qp_max_recv_dtos_mask; 1080 my_rx_head = (my_rx_head + 1) & iser_conn->qp_max_recv_dtos_mask;
1041 } 1081 }
1042 1082
1043 rx_wr--; 1083 rx_wr--;
@@ -1049,7 +1089,7 @@ int iser_post_recvm(struct iser_conn *ib_conn, int count)
1049 iser_err("ib_post_recv failed ret=%d\n", ib_ret); 1089 iser_err("ib_post_recv failed ret=%d\n", ib_ret);
1050 ib_conn->post_recv_buf_count -= count; 1090 ib_conn->post_recv_buf_count -= count;
1051 } else 1091 } else
1052 ib_conn->rx_desc_head = my_rx_head; 1092 iser_conn->rx_desc_head = my_rx_head;
1053 return ib_ret; 1093 return ib_ret;
1054} 1094}
1055 1095
@@ -1059,139 +1099,166 @@ int iser_post_recvm(struct iser_conn *ib_conn, int count)
1059 * 1099 *
1060 * returns 0 on success, -1 on failure 1100 * returns 0 on success, -1 on failure
1061 */ 1101 */
1062int iser_post_send(struct iser_conn *ib_conn, struct iser_tx_desc *tx_desc) 1102int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
1103 bool signal)
1063{ 1104{
1064 int ib_ret; 1105 int ib_ret;
1065 struct ib_send_wr send_wr, *send_wr_failed; 1106 struct ib_send_wr send_wr, *send_wr_failed;
1066 1107
1067 ib_dma_sync_single_for_device(ib_conn->device->ib_device, 1108 ib_dma_sync_single_for_device(ib_conn->device->ib_device,
1068 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 1109 tx_desc->dma_addr, ISER_HEADERS_LEN,
1110 DMA_TO_DEVICE);
1069 1111
1070 send_wr.next = NULL; 1112 send_wr.next = NULL;
1071 send_wr.wr_id = (unsigned long)tx_desc; 1113 send_wr.wr_id = (unsigned long)tx_desc;
1072 send_wr.sg_list = tx_desc->tx_sg; 1114 send_wr.sg_list = tx_desc->tx_sg;
1073 send_wr.num_sge = tx_desc->num_sge; 1115 send_wr.num_sge = tx_desc->num_sge;
1074 send_wr.opcode = IB_WR_SEND; 1116 send_wr.opcode = IB_WR_SEND;
1075 send_wr.send_flags = IB_SEND_SIGNALED; 1117 send_wr.send_flags = signal ? IB_SEND_SIGNALED : 0;
1076
1077 atomic_inc(&ib_conn->post_send_buf_count);
1078 1118
1079 ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed); 1119 ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed);
1080 if (ib_ret) { 1120 if (ib_ret)
1081 iser_err("ib_post_send failed, ret:%d\n", ib_ret); 1121 iser_err("ib_post_send failed, ret:%d\n", ib_ret);
1082 atomic_dec(&ib_conn->post_send_buf_count); 1122
1083 }
1084 return ib_ret; 1123 return ib_ret;
1085} 1124}
1086 1125
1087static void iser_handle_comp_error(struct iser_tx_desc *desc, 1126/**
1088 struct iser_conn *ib_conn) 1127 * is_iser_tx_desc - Indicate if the completion wr_id
1128 * is a TX descriptor or not.
1129 * @iser_conn: iser connection
1130 * @wr_id: completion WR identifier
1131 *
1132 * Since we cannot rely on wc opcode in FLUSH errors
1133 * we must work around it by checking if the wr_id address
1134 * falls in the iser connection rx_descs buffer. If so
1135 * it is an RX descriptor, otherwize it is a TX.
1136 */
1137static inline bool
1138is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id)
1139{
1140 void *start = iser_conn->rx_descs;
1141 int len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs);
1142
1143 if (wr_id >= start && wr_id < start + len)
1144 return false;
1145
1146 return true;
1147}
1148
1149/**
1150 * iser_handle_comp_error() - Handle error completion
1151 * @ib_conn: connection RDMA resources
1152 * @wc: work completion
1153 *
1154 * Notes: We may handle a FLUSH error completion and in this case
1155 * we only cleanup in case TX type was DATAOUT. For non-FLUSH
1156 * error completion we should also notify iscsi layer that
1157 * connection is failed (in case we passed bind stage).
1158 */
1159static void
1160iser_handle_comp_error(struct ib_conn *ib_conn,
1161 struct ib_wc *wc)
1089{ 1162{
1090 if (desc && desc->type == ISCSI_TX_DATAOUT) 1163 struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
1091 kmem_cache_free(ig.desc_cache, desc); 1164 ib_conn);
1092 1165
1093 if (ib_conn->post_recv_buf_count == 0 && 1166 if (wc->status != IB_WC_WR_FLUSH_ERR)
1094 atomic_read(&ib_conn->post_send_buf_count) == 0) { 1167 if (iser_conn->iscsi_conn)
1095 /** 1168 iscsi_conn_failure(iser_conn->iscsi_conn,
1096 * getting here when the state is UP means that the conn is
1097 * being terminated asynchronously from the iSCSI layer's
1098 * perspective. It is safe to peek at the connection state
1099 * since iscsi_conn_failure is allowed to be called twice.
1100 **/
1101 if (ib_conn->state == ISER_CONN_UP)
1102 iscsi_conn_failure(ib_conn->iscsi_conn,
1103 ISCSI_ERR_CONN_FAILED); 1169 ISCSI_ERR_CONN_FAILED);
1104 1170
1105 /* no more non completed posts to the QP, complete the 1171 if (is_iser_tx_desc(iser_conn, (void *)wc->wr_id)) {
1106 * termination process w.o worrying on disconnect event */ 1172 struct iser_tx_desc *desc = (struct iser_tx_desc *)wc->wr_id;
1107 complete(&ib_conn->flush_completion); 1173
1174 if (desc->type == ISCSI_TX_DATAOUT)
1175 kmem_cache_free(ig.desc_cache, desc);
1176 } else {
1177 ib_conn->post_recv_buf_count--;
1108 } 1178 }
1109} 1179}
1110 1180
1111static int iser_drain_tx_cq(struct iser_device *device, int cq_index) 1181/**
1182 * iser_handle_wc - handle a single work completion
1183 * @wc: work completion
1184 *
1185 * Soft-IRQ context, work completion can be either
1186 * SEND or RECV, and can turn out successful or
1187 * with error (or flush error).
1188 */
1189static void iser_handle_wc(struct ib_wc *wc)
1112{ 1190{
1113 struct ib_cq *cq = device->tx_cq[cq_index]; 1191 struct ib_conn *ib_conn;
1114 struct ib_wc wc;
1115 struct iser_tx_desc *tx_desc; 1192 struct iser_tx_desc *tx_desc;
1116 struct iser_conn *ib_conn; 1193 struct iser_rx_desc *rx_desc;
1117 int completed_tx = 0; 1194
1118 1195 ib_conn = wc->qp->qp_context;
1119 while (ib_poll_cq(cq, 1, &wc) == 1) { 1196 if (wc->status == IB_WC_SUCCESS) {
1120 tx_desc = (struct iser_tx_desc *) (unsigned long) wc.wr_id; 1197 if (wc->opcode == IB_WC_RECV) {
1121 ib_conn = wc.qp->qp_context; 1198 rx_desc = (struct iser_rx_desc *)wc->wr_id;
1122 if (wc.status == IB_WC_SUCCESS) { 1199 iser_rcv_completion(rx_desc, wc->byte_len,
1123 if (wc.opcode == IB_WC_SEND) 1200 ib_conn);
1124 iser_snd_completion(tx_desc, ib_conn); 1201 } else
1125 else 1202 if (wc->opcode == IB_WC_SEND) {
1126 iser_err("expected opcode %d got %d\n", 1203 tx_desc = (struct iser_tx_desc *)wc->wr_id;
1127 IB_WC_SEND, wc.opcode); 1204 iser_snd_completion(tx_desc, ib_conn);
1128 } else { 1205 } else {
1129 iser_err("tx id %llx status %d vend_err %x\n", 1206 iser_err("Unknown wc opcode %d\n", wc->opcode);
1130 wc.wr_id, wc.status, wc.vendor_err);
1131 if (wc.wr_id != ISER_FASTREG_LI_WRID) {
1132 atomic_dec(&ib_conn->post_send_buf_count);
1133 iser_handle_comp_error(tx_desc, ib_conn);
1134 }
1135 } 1207 }
1136 completed_tx++; 1208 } else {
1209 if (wc->status != IB_WC_WR_FLUSH_ERR)
1210 iser_err("wr id %llx status %d vend_err %x\n",
1211 wc->wr_id, wc->status, wc->vendor_err);
1212 else
1213 iser_dbg("flush error: wr id %llx\n", wc->wr_id);
1214
1215 if (wc->wr_id != ISER_FASTREG_LI_WRID &&
1216 wc->wr_id != ISER_BEACON_WRID)
1217 iser_handle_comp_error(ib_conn, wc);
1218
1219 /* complete in case all flush errors were consumed */
1220 if (wc->wr_id == ISER_BEACON_WRID)
1221 complete(&ib_conn->flush_comp);
1137 } 1222 }
1138 return completed_tx;
1139} 1223}
1140 1224
1141 1225/**
1226 * iser_cq_tasklet_fn - iSER completion polling loop
1227 * @data: iSER completion context
1228 *
1229 * Soft-IRQ context, polling connection CQ until
1230 * either CQ was empty or we exausted polling budget
1231 */
1142static void iser_cq_tasklet_fn(unsigned long data) 1232static void iser_cq_tasklet_fn(unsigned long data)
1143{ 1233{
1144 struct iser_cq_desc *cq_desc = (struct iser_cq_desc *)data; 1234 struct iser_comp *comp = (struct iser_comp *)data;
1145 struct iser_device *device = cq_desc->device; 1235 struct ib_cq *cq = comp->cq;
1146 int cq_index = cq_desc->cq_index; 1236 struct ib_wc *const wcs = comp->wcs;
1147 struct ib_cq *cq = device->rx_cq[cq_index]; 1237 int i, n, completed = 0;
1148 struct ib_wc wc; 1238
1149 struct iser_rx_desc *desc; 1239 while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) {
1150 unsigned long xfer_len; 1240 for (i = 0; i < n; i++)
1151 struct iser_conn *ib_conn; 1241 iser_handle_wc(&wcs[i]);
1152 int completed_tx, completed_rx = 0; 1242
1153 1243 completed += n;
1154 /* First do tx drain, so in a case where we have rx flushes and a successful 1244 if (completed >= iser_cq_poll_limit)
1155 * tx completion we will still go through completion error handling. 1245 break;
1156 */
1157 completed_tx = iser_drain_tx_cq(device, cq_index);
1158
1159 while (ib_poll_cq(cq, 1, &wc) == 1) {
1160 desc = (struct iser_rx_desc *) (unsigned long) wc.wr_id;
1161 BUG_ON(desc == NULL);
1162 ib_conn = wc.qp->qp_context;
1163 if (wc.status == IB_WC_SUCCESS) {
1164 if (wc.opcode == IB_WC_RECV) {
1165 xfer_len = (unsigned long)wc.byte_len;
1166 iser_rcv_completion(desc, xfer_len, ib_conn);
1167 } else
1168 iser_err("expected opcode %d got %d\n",
1169 IB_WC_RECV, wc.opcode);
1170 } else {
1171 if (wc.status != IB_WC_WR_FLUSH_ERR)
1172 iser_err("rx id %llx status %d vend_err %x\n",
1173 wc.wr_id, wc.status, wc.vendor_err);
1174 ib_conn->post_recv_buf_count--;
1175 iser_handle_comp_error(NULL, ib_conn);
1176 }
1177 completed_rx++;
1178 if (!(completed_rx & 63))
1179 completed_tx += iser_drain_tx_cq(device, cq_index);
1180 } 1246 }
1181 /* #warning "it is assumed here that arming CQ only once its empty" * 1247
1182 * " would not cause interrupts to be missed" */ 1248 /*
1249 * It is assumed here that arming CQ only once its empty
1250 * would not cause interrupts to be missed.
1251 */
1183 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); 1252 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1184 1253
1185 iser_dbg("got %d rx %d tx completions\n", completed_rx, completed_tx); 1254 iser_dbg("got %d completions\n", completed);
1186} 1255}
1187 1256
1188static void iser_cq_callback(struct ib_cq *cq, void *cq_context) 1257static void iser_cq_callback(struct ib_cq *cq, void *cq_context)
1189{ 1258{
1190 struct iser_cq_desc *cq_desc = (struct iser_cq_desc *)cq_context; 1259 struct iser_comp *comp = cq_context;
1191 struct iser_device *device = cq_desc->device;
1192 int cq_index = cq_desc->cq_index;
1193 1260
1194 tasklet_schedule(&device->cq_tasklet[cq_index]); 1261 tasklet_schedule(&comp->tasklet);
1195} 1262}
1196 1263
1197u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task, 1264u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index da8ff124762a..0bea5776bcbc 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -2609,58 +2609,45 @@ isert_fast_reg_mr(struct isert_conn *isert_conn,
2609 return ret; 2609 return ret;
2610} 2610}
2611 2611
2612static inline enum ib_t10_dif_type 2612static inline void
2613se2ib_prot_type(enum target_prot_type prot_type) 2613isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs,
2614{ 2614 struct ib_sig_domain *domain)
2615 switch (prot_type) { 2615{
2616 case TARGET_DIF_TYPE0_PROT: 2616 domain->sig_type = IB_SIG_TYPE_T10_DIF;
2617 return IB_T10DIF_NONE; 2617 domain->sig.dif.bg_type = IB_T10DIF_CRC;
2618 case TARGET_DIF_TYPE1_PROT: 2618 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
2619 return IB_T10DIF_TYPE1; 2619 domain->sig.dif.ref_tag = se_cmd->reftag_seed;
2620 case TARGET_DIF_TYPE2_PROT: 2620 /*
2621 return IB_T10DIF_TYPE2; 2621 * At the moment we hard code those, but if in the future
2622 case TARGET_DIF_TYPE3_PROT: 2622 * the target core would like to use it, we will take it
2623 return IB_T10DIF_TYPE3; 2623 * from se_cmd.
2624 default: 2624 */
2625 return IB_T10DIF_NONE; 2625 domain->sig.dif.apptag_check_mask = 0xffff;
2626 } 2626 domain->sig.dif.app_escape = true;
2627} 2627 domain->sig.dif.ref_escape = true;
2628 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
2629 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
2630 domain->sig.dif.ref_remap = true;
2631};
2628 2632
2629static int 2633static int
2630isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2634isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
2631{ 2635{
2632 enum ib_t10_dif_type ib_prot_type = se2ib_prot_type(se_cmd->prot_type);
2633
2634 sig_attrs->mem.sig_type = IB_SIG_TYPE_T10_DIF;
2635 sig_attrs->wire.sig_type = IB_SIG_TYPE_T10_DIF;
2636 sig_attrs->mem.sig.dif.pi_interval =
2637 se_cmd->se_dev->dev_attrib.block_size;
2638 sig_attrs->wire.sig.dif.pi_interval =
2639 se_cmd->se_dev->dev_attrib.block_size;
2640
2641 switch (se_cmd->prot_op) { 2636 switch (se_cmd->prot_op) {
2642 case TARGET_PROT_DIN_INSERT: 2637 case TARGET_PROT_DIN_INSERT:
2643 case TARGET_PROT_DOUT_STRIP: 2638 case TARGET_PROT_DOUT_STRIP:
2644 sig_attrs->mem.sig.dif.type = IB_T10DIF_NONE; 2639 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
2645 sig_attrs->wire.sig.dif.type = ib_prot_type; 2640 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
2646 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
2647 sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed;
2648 break; 2641 break;
2649 case TARGET_PROT_DOUT_INSERT: 2642 case TARGET_PROT_DOUT_INSERT:
2650 case TARGET_PROT_DIN_STRIP: 2643 case TARGET_PROT_DIN_STRIP:
2651 sig_attrs->mem.sig.dif.type = ib_prot_type; 2644 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
2652 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 2645 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
2653 sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed;
2654 sig_attrs->wire.sig.dif.type = IB_T10DIF_NONE;
2655 break; 2646 break;
2656 case TARGET_PROT_DIN_PASS: 2647 case TARGET_PROT_DIN_PASS:
2657 case TARGET_PROT_DOUT_PASS: 2648 case TARGET_PROT_DOUT_PASS:
2658 sig_attrs->mem.sig.dif.type = ib_prot_type; 2649 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
2659 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 2650 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
2660 sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed;
2661 sig_attrs->wire.sig.dif.type = ib_prot_type;
2662 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC;
2663 sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed;
2664 break; 2651 break;
2665 default: 2652 default:
2666 pr_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2653 pr_err("Unsupported PI operation %d\n", se_cmd->prot_op);
diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
index 7c4c0f1f5805..3fa075daeb1d 100644
--- a/include/linux/mlx5/qp.h
+++ b/include/linux/mlx5/qp.h
@@ -40,6 +40,15 @@
40#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5) 40#define MLX5_SIG_WQE_SIZE (MLX5_SEND_WQE_BB * 5)
41#define MLX5_DIF_SIZE 8 41#define MLX5_DIF_SIZE 8
42#define MLX5_STRIDE_BLOCK_OP 0x400 42#define MLX5_STRIDE_BLOCK_OP 0x400
43#define MLX5_CPY_GRD_MASK 0xc0
44#define MLX5_CPY_APP_MASK 0x30
45#define MLX5_CPY_REF_MASK 0x0f
46#define MLX5_BSF_INC_REFTAG (1 << 6)
47#define MLX5_BSF_INL_VALID (1 << 15)
48#define MLX5_BSF_REFRESH_DIF (1 << 14)
49#define MLX5_BSF_REPEAT_BLOCK (1 << 7)
50#define MLX5_BSF_APPTAG_ESCAPE 0x1
51#define MLX5_BSF_APPREF_ESCAPE 0x2
43 52
44enum mlx5_qp_optpar { 53enum mlx5_qp_optpar {
45 MLX5_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0, 54 MLX5_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0,
@@ -287,6 +296,22 @@ struct mlx5_wqe_inline_seg {
287 __be32 byte_count; 296 __be32 byte_count;
288}; 297};
289 298
299enum mlx5_sig_type {
300 MLX5_DIF_CRC = 0x1,
301 MLX5_DIF_IPCS = 0x2,
302};
303
304struct mlx5_bsf_inl {
305 __be16 vld_refresh;
306 __be16 dif_apptag;
307 __be32 dif_reftag;
308 u8 sig_type;
309 u8 rp_inv_seed;
310 u8 rsvd[3];
311 u8 dif_inc_ref_guard_check;
312 __be16 dif_app_bitmask_check;
313};
314
290struct mlx5_bsf { 315struct mlx5_bsf {
291 struct mlx5_bsf_basic { 316 struct mlx5_bsf_basic {
292 u8 bsf_size_sbs; 317 u8 bsf_size_sbs;
@@ -310,14 +335,8 @@ struct mlx5_bsf {
310 __be32 w_tfs_psv; 335 __be32 w_tfs_psv;
311 __be32 m_tfs_psv; 336 __be32 m_tfs_psv;
312 } ext; 337 } ext;
313 struct mlx5_bsf_inl { 338 struct mlx5_bsf_inl w_inl;
314 __be32 w_inl_vld; 339 struct mlx5_bsf_inl m_inl;
315 __be32 w_rsvd;
316 __be64 w_block_format;
317 __be32 m_inl_vld;
318 __be32 m_rsvd;
319 __be64 m_block_format;
320 } inl;
321}; 340};
322 341
323struct mlx5_klm { 342struct mlx5_klm {
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index ed44cc07a7b3..470a011d6fa4 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -491,20 +491,14 @@ struct ib_mr_init_attr {
491 u32 flags; 491 u32 flags;
492}; 492};
493 493
494enum ib_signature_type {
495 IB_SIG_TYPE_T10_DIF,
496};
497
498/** 494/**
499 * T10-DIF Signature types 495 * Signature types
500 * T10-DIF types are defined by SCSI 496 * IB_SIG_TYPE_NONE: Unprotected.
501 * specifications. 497 * IB_SIG_TYPE_T10_DIF: Type T10-DIF
502 */ 498 */
503enum ib_t10_dif_type { 499enum ib_signature_type {
504 IB_T10DIF_NONE, 500 IB_SIG_TYPE_NONE,
505 IB_T10DIF_TYPE1, 501 IB_SIG_TYPE_T10_DIF,
506 IB_T10DIF_TYPE2,
507 IB_T10DIF_TYPE3
508}; 502};
509 503
510/** 504/**
@@ -520,24 +514,26 @@ enum ib_t10_dif_bg_type {
520/** 514/**
521 * struct ib_t10_dif_domain - Parameters specific for T10-DIF 515 * struct ib_t10_dif_domain - Parameters specific for T10-DIF
522 * domain. 516 * domain.
523 * @type: T10-DIF type (0|1|2|3)
524 * @bg_type: T10-DIF block guard type (CRC|CSUM) 517 * @bg_type: T10-DIF block guard type (CRC|CSUM)
525 * @pi_interval: protection information interval. 518 * @pi_interval: protection information interval.
526 * @bg: seed of guard computation. 519 * @bg: seed of guard computation.
527 * @app_tag: application tag of guard block 520 * @app_tag: application tag of guard block
528 * @ref_tag: initial guard block reference tag. 521 * @ref_tag: initial guard block reference tag.
529 * @type3_inc_reftag: T10-DIF type 3 does not state 522 * @ref_remap: Indicate wethear the reftag increments each block
530 * about the reference tag, it is the user 523 * @app_escape: Indicate to skip block check if apptag=0xffff
531 * choice to increment it or not. 524 * @ref_escape: Indicate to skip block check if reftag=0xffffffff
525 * @apptag_check_mask: check bitmask of application tag.
532 */ 526 */
533struct ib_t10_dif_domain { 527struct ib_t10_dif_domain {
534 enum ib_t10_dif_type type;
535 enum ib_t10_dif_bg_type bg_type; 528 enum ib_t10_dif_bg_type bg_type;
536 u16 pi_interval; 529 u16 pi_interval;
537 u16 bg; 530 u16 bg;
538 u16 app_tag; 531 u16 app_tag;
539 u32 ref_tag; 532 u32 ref_tag;
540 bool type3_inc_reftag; 533 bool ref_remap;
534 bool app_escape;
535 bool ref_escape;
536 u16 apptag_check_mask;
541}; 537};
542 538
543/** 539/**
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-04 16:40:23 -0400