aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/infiniband/ulp
diff options
context:
space:
mode:
authorDavid S. Miller <davem@davemloft.net>2008-07-18 05:39:39 -0400
committerDavid S. Miller <davem@davemloft.net>2008-07-18 05:39:39 -0400
commit49997d75152b3d23c53b0fa730599f2f74c92c65 (patch)
tree46e93126170d02cfec9505172e545732c1b69656 /drivers/infiniband/ulp
parenta0c80b80e0fb48129e4e9d6a9ede914f9ff1850d (diff)
parent5b664cb235e97afbf34db9c4d77f08ebd725335e (diff)
Merge branch 'master' of master.kernel.org:/pub/scm/linux/kernel/git/torvalds/linux-2.6
Conflicts: Documentation/powerpc/booting-without-of.txt drivers/atm/Makefile drivers/net/fs_enet/fs_enet-main.c drivers/pci/pci-acpi.c net/8021q/vlan.c net/iucv/iucv.c
Diffstat (limited to 'drivers/infiniband/ulp')
-rw-r--r--drivers/infiniband/ulp/ipoib/Kconfig1
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib.h48
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_cm.c104
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_ethtool.c46
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_fs.c2
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_ib.c52
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_main.c115
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_multicast.c27
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_verbs.c69
-rw-r--r--drivers/infiniband/ulp/ipoib/ipoib_vlan.c2
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.c359
-rw-r--r--drivers/infiniband/ulp/iser/iscsi_iser.h46
-rw-r--r--drivers/infiniband/ulp/iser/iser_initiator.c211
-rw-r--r--drivers/infiniband/ulp/iser/iser_memory.c79
-rw-r--r--drivers/infiniband/ulp/iser/iser_verbs.c30
-rw-r--r--drivers/infiniband/ulp/srp/ib_srp.c15
-rw-r--r--drivers/infiniband/ulp/srp/ib_srp.h2
17 files changed, 727 insertions, 481 deletions
diff --git a/drivers/infiniband/ulp/ipoib/Kconfig b/drivers/infiniband/ulp/ipoib/Kconfig
index 1f76bad020f3..691525cf394a 100644
--- a/drivers/infiniband/ulp/ipoib/Kconfig
+++ b/drivers/infiniband/ulp/ipoib/Kconfig
@@ -1,6 +1,7 @@
1config INFINIBAND_IPOIB 1config INFINIBAND_IPOIB
2 tristate "IP-over-InfiniBand" 2 tristate "IP-over-InfiniBand"
3 depends on NETDEVICES && INET && (IPV6 || IPV6=n) 3 depends on NETDEVICES && INET && (IPV6 || IPV6=n)
4 select INET_LRO
4 ---help--- 5 ---help---
5 Support for the IP-over-InfiniBand protocol (IPoIB). This 6 Support for the IP-over-InfiniBand protocol (IPoIB). This
6 transports IP packets over InfiniBand so you can use your IB 7 transports IP packets over InfiniBand so you can use your IB
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index ca126fc2b853..b0ffc9abe8c0 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -30,8 +30,6 @@
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE. 32 * SOFTWARE.
33 *
34 * $Id: ipoib.h 1358 2004-12-17 22:00:11Z roland $
35 */ 33 */
36 34
37#ifndef _IPOIB_H 35#ifndef _IPOIB_H
@@ -52,9 +50,16 @@
52#include <rdma/ib_verbs.h> 50#include <rdma/ib_verbs.h>
53#include <rdma/ib_pack.h> 51#include <rdma/ib_pack.h>
54#include <rdma/ib_sa.h> 52#include <rdma/ib_sa.h>
53#include <linux/inet_lro.h>
55 54
56/* constants */ 55/* constants */
57 56
57enum ipoib_flush_level {
58 IPOIB_FLUSH_LIGHT,
59 IPOIB_FLUSH_NORMAL,
60 IPOIB_FLUSH_HEAVY
61};
62
58enum { 63enum {
59 IPOIB_ENCAP_LEN = 4, 64 IPOIB_ENCAP_LEN = 4,
60 65
@@ -65,8 +70,8 @@ enum {
65 IPOIB_CM_BUF_SIZE = IPOIB_CM_MTU + IPOIB_ENCAP_LEN, 70 IPOIB_CM_BUF_SIZE = IPOIB_CM_MTU + IPOIB_ENCAP_LEN,
66 IPOIB_CM_HEAD_SIZE = IPOIB_CM_BUF_SIZE % PAGE_SIZE, 71 IPOIB_CM_HEAD_SIZE = IPOIB_CM_BUF_SIZE % PAGE_SIZE,
67 IPOIB_CM_RX_SG = ALIGN(IPOIB_CM_BUF_SIZE, PAGE_SIZE) / PAGE_SIZE, 72 IPOIB_CM_RX_SG = ALIGN(IPOIB_CM_BUF_SIZE, PAGE_SIZE) / PAGE_SIZE,
68 IPOIB_RX_RING_SIZE = 128, 73 IPOIB_RX_RING_SIZE = 256,
69 IPOIB_TX_RING_SIZE = 64, 74 IPOIB_TX_RING_SIZE = 128,
70 IPOIB_MAX_QUEUE_SIZE = 8192, 75 IPOIB_MAX_QUEUE_SIZE = 8192,
71 IPOIB_MIN_QUEUE_SIZE = 2, 76 IPOIB_MIN_QUEUE_SIZE = 2,
72 IPOIB_CM_MAX_CONN_QP = 4096, 77 IPOIB_CM_MAX_CONN_QP = 4096,
@@ -84,7 +89,6 @@ enum {
84 IPOIB_FLAG_SUBINTERFACE = 5, 89 IPOIB_FLAG_SUBINTERFACE = 5,
85 IPOIB_MCAST_RUN = 6, 90 IPOIB_MCAST_RUN = 6,
86 IPOIB_STOP_REAPER = 7, 91 IPOIB_STOP_REAPER = 7,
87 IPOIB_MCAST_STARTED = 8,
88 IPOIB_FLAG_ADMIN_CM = 9, 92 IPOIB_FLAG_ADMIN_CM = 9,
89 IPOIB_FLAG_UMCAST = 10, 93 IPOIB_FLAG_UMCAST = 10,
90 IPOIB_FLAG_CSUM = 11, 94 IPOIB_FLAG_CSUM = 11,
@@ -96,7 +100,11 @@ enum {
96 IPOIB_MCAST_FLAG_BUSY = 2, /* joining or already joined */ 100 IPOIB_MCAST_FLAG_BUSY = 2, /* joining or already joined */
97 IPOIB_MCAST_FLAG_ATTACHED = 3, 101 IPOIB_MCAST_FLAG_ATTACHED = 3,
98 102
103 IPOIB_MAX_LRO_DESCRIPTORS = 8,
104 IPOIB_LRO_MAX_AGGR = 64,
105
99 MAX_SEND_CQE = 16, 106 MAX_SEND_CQE = 16,
107 IPOIB_CM_COPYBREAK = 256,
100}; 108};
101 109
102#define IPOIB_OP_RECV (1ul << 31) 110#define IPOIB_OP_RECV (1ul << 31)
@@ -149,6 +157,11 @@ struct ipoib_tx_buf {
149 u64 mapping[MAX_SKB_FRAGS + 1]; 157 u64 mapping[MAX_SKB_FRAGS + 1];
150}; 158};
151 159
160struct ipoib_cm_tx_buf {
161 struct sk_buff *skb;
162 u64 mapping;
163};
164
152struct ib_cm_id; 165struct ib_cm_id;
153 166
154struct ipoib_cm_data { 167struct ipoib_cm_data {
@@ -207,7 +220,7 @@ struct ipoib_cm_tx {
207 struct net_device *dev; 220 struct net_device *dev;
208 struct ipoib_neigh *neigh; 221 struct ipoib_neigh *neigh;
209 struct ipoib_path *path; 222 struct ipoib_path *path;
210 struct ipoib_tx_buf *tx_ring; 223 struct ipoib_cm_tx_buf *tx_ring;
211 unsigned tx_head; 224 unsigned tx_head;
212 unsigned tx_tail; 225 unsigned tx_tail;
213 unsigned long flags; 226 unsigned long flags;
@@ -249,6 +262,11 @@ struct ipoib_ethtool_st {
249 u16 max_coalesced_frames; 262 u16 max_coalesced_frames;
250}; 263};
251 264
265struct ipoib_lro {
266 struct net_lro_mgr lro_mgr;
267 struct net_lro_desc lro_desc[IPOIB_MAX_LRO_DESCRIPTORS];
268};
269
252/* 270/*
253 * Device private locking: tx_lock protects members used in TX fast 271 * Device private locking: tx_lock protects members used in TX fast
254 * path (and we use LLTX so upper layers don't do extra locking). 272 * path (and we use LLTX so upper layers don't do extra locking).
@@ -264,7 +282,6 @@ struct ipoib_dev_priv {
264 282
265 unsigned long flags; 283 unsigned long flags;
266 284
267 struct mutex mcast_mutex;
268 struct mutex vlan_mutex; 285 struct mutex vlan_mutex;
269 286
270 struct rb_root path_tree; 287 struct rb_root path_tree;
@@ -276,10 +293,11 @@ struct ipoib_dev_priv {
276 293
277 struct delayed_work pkey_poll_task; 294 struct delayed_work pkey_poll_task;
278 struct delayed_work mcast_task; 295 struct delayed_work mcast_task;
279 struct work_struct flush_task; 296 struct work_struct flush_light;
297 struct work_struct flush_normal;
298 struct work_struct flush_heavy;
280 struct work_struct restart_task; 299 struct work_struct restart_task;
281 struct delayed_work ah_reap_task; 300 struct delayed_work ah_reap_task;
282 struct work_struct pkey_event_task;
283 301
284 struct ib_device *ca; 302 struct ib_device *ca;
285 u8 port; 303 u8 port;
@@ -335,6 +353,8 @@ struct ipoib_dev_priv {
335 int hca_caps; 353 int hca_caps;
336 struct ipoib_ethtool_st ethtool; 354 struct ipoib_ethtool_st ethtool;
337 struct timer_list poll_timer; 355 struct timer_list poll_timer;
356
357 struct ipoib_lro lro;
338}; 358};
339 359
340struct ipoib_ah { 360struct ipoib_ah {
@@ -359,6 +379,7 @@ struct ipoib_path {
359 379
360 struct rb_node rb_node; 380 struct rb_node rb_node;
361 struct list_head list; 381 struct list_head list;
382 int valid;
362}; 383};
363 384
364struct ipoib_neigh { 385struct ipoib_neigh {
@@ -423,11 +444,14 @@ void ipoib_send(struct net_device *dev, struct sk_buff *skb,
423 struct ipoib_ah *address, u32 qpn); 444 struct ipoib_ah *address, u32 qpn);
424void ipoib_reap_ah(struct work_struct *work); 445void ipoib_reap_ah(struct work_struct *work);
425 446
447void ipoib_mark_paths_invalid(struct net_device *dev);
426void ipoib_flush_paths(struct net_device *dev); 448void ipoib_flush_paths(struct net_device *dev);
427struct ipoib_dev_priv *ipoib_intf_alloc(const char *format); 449struct ipoib_dev_priv *ipoib_intf_alloc(const char *format);
428 450
429int ipoib_ib_dev_init(struct net_device *dev, struct ib_device *ca, int port); 451int ipoib_ib_dev_init(struct net_device *dev, struct ib_device *ca, int port);
430void ipoib_ib_dev_flush(struct work_struct *work); 452void ipoib_ib_dev_flush_light(struct work_struct *work);
453void ipoib_ib_dev_flush_normal(struct work_struct *work);
454void ipoib_ib_dev_flush_heavy(struct work_struct *work);
431void ipoib_pkey_event(struct work_struct *work); 455void ipoib_pkey_event(struct work_struct *work);
432void ipoib_ib_dev_cleanup(struct net_device *dev); 456void ipoib_ib_dev_cleanup(struct net_device *dev);
433 457
@@ -466,9 +490,7 @@ void ipoib_path_iter_read(struct ipoib_path_iter *iter,
466#endif 490#endif
467 491
468int ipoib_mcast_attach(struct net_device *dev, u16 mlid, 492int ipoib_mcast_attach(struct net_device *dev, u16 mlid,
469 union ib_gid *mgid); 493 union ib_gid *mgid, int set_qkey);
470int ipoib_mcast_detach(struct net_device *dev, u16 mlid,
471 union ib_gid *mgid);
472 494
473int ipoib_init_qp(struct net_device *dev); 495int ipoib_init_qp(struct net_device *dev);
474int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca); 496int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca);
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_cm.c b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
index 97e67d36378f..0f2d3045061a 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_cm.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_cm.c
@@ -28,8 +28,6 @@
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE. 30 * SOFTWARE.
31 *
32 * $Id$
33 */ 31 */
34 32
35#include <rdma/ib_cm.h> 33#include <rdma/ib_cm.h>
@@ -113,18 +111,20 @@ static int ipoib_cm_post_receive_srq(struct net_device *dev, int id)
113} 111}
114 112
115static int ipoib_cm_post_receive_nonsrq(struct net_device *dev, 113static int ipoib_cm_post_receive_nonsrq(struct net_device *dev,
116 struct ipoib_cm_rx *rx, int id) 114 struct ipoib_cm_rx *rx,
115 struct ib_recv_wr *wr,
116 struct ib_sge *sge, int id)
117{ 117{
118 struct ipoib_dev_priv *priv = netdev_priv(dev); 118 struct ipoib_dev_priv *priv = netdev_priv(dev);
119 struct ib_recv_wr *bad_wr; 119 struct ib_recv_wr *bad_wr;
120 int i, ret; 120 int i, ret;
121 121
122 priv->cm.rx_wr.wr_id = id | IPOIB_OP_CM | IPOIB_OP_RECV; 122 wr->wr_id = id | IPOIB_OP_CM | IPOIB_OP_RECV;
123 123
124 for (i = 0; i < IPOIB_CM_RX_SG; ++i) 124 for (i = 0; i < IPOIB_CM_RX_SG; ++i)
125 priv->cm.rx_sge[i].addr = rx->rx_ring[id].mapping[i]; 125 sge[i].addr = rx->rx_ring[id].mapping[i];
126 126
127 ret = ib_post_recv(rx->qp, &priv->cm.rx_wr, &bad_wr); 127 ret = ib_post_recv(rx->qp, wr, &bad_wr);
128 if (unlikely(ret)) { 128 if (unlikely(ret)) {
129 ipoib_warn(priv, "post recv failed for buf %d (%d)\n", id, ret); 129 ipoib_warn(priv, "post recv failed for buf %d (%d)\n", id, ret);
130 ipoib_cm_dma_unmap_rx(priv, IPOIB_CM_RX_SG - 1, 130 ipoib_cm_dma_unmap_rx(priv, IPOIB_CM_RX_SG - 1,
@@ -322,10 +322,33 @@ static int ipoib_cm_modify_rx_qp(struct net_device *dev,
322 return 0; 322 return 0;
323} 323}
324 324
325static void ipoib_cm_init_rx_wr(struct net_device *dev,
326 struct ib_recv_wr *wr,
327 struct ib_sge *sge)
328{
329 struct ipoib_dev_priv *priv = netdev_priv(dev);
330 int i;
331
332 for (i = 0; i < priv->cm.num_frags; ++i)
333 sge[i].lkey = priv->mr->lkey;
334
335 sge[0].length = IPOIB_CM_HEAD_SIZE;
336 for (i = 1; i < priv->cm.num_frags; ++i)
337 sge[i].length = PAGE_SIZE;
338
339 wr->next = NULL;
340 wr->sg_list = priv->cm.rx_sge;
341 wr->num_sge = priv->cm.num_frags;
342}
343
325static int ipoib_cm_nonsrq_init_rx(struct net_device *dev, struct ib_cm_id *cm_id, 344static int ipoib_cm_nonsrq_init_rx(struct net_device *dev, struct ib_cm_id *cm_id,
326 struct ipoib_cm_rx *rx) 345 struct ipoib_cm_rx *rx)
327{ 346{
328 struct ipoib_dev_priv *priv = netdev_priv(dev); 347 struct ipoib_dev_priv *priv = netdev_priv(dev);
348 struct {
349 struct ib_recv_wr wr;
350 struct ib_sge sge[IPOIB_CM_RX_SG];
351 } *t;
329 int ret; 352 int ret;
330 int i; 353 int i;
331 354
@@ -333,6 +356,14 @@ static int ipoib_cm_nonsrq_init_rx(struct net_device *dev, struct ib_cm_id *cm_i
333 if (!rx->rx_ring) 356 if (!rx->rx_ring)
334 return -ENOMEM; 357 return -ENOMEM;
335 358
359 t = kmalloc(sizeof *t, GFP_KERNEL);
360 if (!t) {
361 ret = -ENOMEM;
362 goto err_free;
363 }
364
365 ipoib_cm_init_rx_wr(dev, &t->wr, t->sge);
366
336 spin_lock_irq(&priv->lock); 367 spin_lock_irq(&priv->lock);
337 368
338 if (priv->cm.nonsrq_conn_qp >= ipoib_max_conn_qp) { 369 if (priv->cm.nonsrq_conn_qp >= ipoib_max_conn_qp) {
@@ -351,8 +382,8 @@ static int ipoib_cm_nonsrq_init_rx(struct net_device *dev, struct ib_cm_id *cm_i
351 ipoib_warn(priv, "failed to allocate receive buffer %d\n", i); 382 ipoib_warn(priv, "failed to allocate receive buffer %d\n", i);
352 ret = -ENOMEM; 383 ret = -ENOMEM;
353 goto err_count; 384 goto err_count;
354 } 385 }
355 ret = ipoib_cm_post_receive_nonsrq(dev, rx, i); 386 ret = ipoib_cm_post_receive_nonsrq(dev, rx, &t->wr, t->sge, i);
356 if (ret) { 387 if (ret) {
357 ipoib_warn(priv, "ipoib_cm_post_receive_nonsrq " 388 ipoib_warn(priv, "ipoib_cm_post_receive_nonsrq "
358 "failed for buf %d\n", i); 389 "failed for buf %d\n", i);
@@ -363,6 +394,8 @@ static int ipoib_cm_nonsrq_init_rx(struct net_device *dev, struct ib_cm_id *cm_i
363 394
364 rx->recv_count = ipoib_recvq_size; 395 rx->recv_count = ipoib_recvq_size;
365 396
397 kfree(t);
398
366 return 0; 399 return 0;
367 400
368err_count: 401err_count:
@@ -371,6 +404,7 @@ err_count:
371 spin_unlock_irq(&priv->lock); 404 spin_unlock_irq(&priv->lock);
372 405
373err_free: 406err_free:
407 kfree(t);
374 ipoib_cm_free_rx_ring(dev, rx->rx_ring); 408 ipoib_cm_free_rx_ring(dev, rx->rx_ring);
375 409
376 return ret; 410 return ret;
@@ -525,6 +559,7 @@ void ipoib_cm_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
525 u64 mapping[IPOIB_CM_RX_SG]; 559 u64 mapping[IPOIB_CM_RX_SG];
526 int frags; 560 int frags;
527 int has_srq; 561 int has_srq;
562 struct sk_buff *small_skb;
528 563
529 ipoib_dbg_data(priv, "cm recv completion: id %d, status: %d\n", 564 ipoib_dbg_data(priv, "cm recv completion: id %d, status: %d\n",
530 wr_id, wc->status); 565 wr_id, wc->status);
@@ -579,6 +614,23 @@ void ipoib_cm_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
579 } 614 }
580 } 615 }
581 616
617 if (wc->byte_len < IPOIB_CM_COPYBREAK) {
618 int dlen = wc->byte_len;
619
620 small_skb = dev_alloc_skb(dlen + 12);
621 if (small_skb) {
622 skb_reserve(small_skb, 12);
623 ib_dma_sync_single_for_cpu(priv->ca, rx_ring[wr_id].mapping[0],
624 dlen, DMA_FROM_DEVICE);
625 skb_copy_from_linear_data(skb, small_skb->data, dlen);
626 ib_dma_sync_single_for_device(priv->ca, rx_ring[wr_id].mapping[0],
627 dlen, DMA_FROM_DEVICE);
628 skb_put(small_skb, dlen);
629 skb = small_skb;
630 goto copied;
631 }
632 }
633
582 frags = PAGE_ALIGN(wc->byte_len - min(wc->byte_len, 634 frags = PAGE_ALIGN(wc->byte_len - min(wc->byte_len,
583 (unsigned)IPOIB_CM_HEAD_SIZE)) / PAGE_SIZE; 635 (unsigned)IPOIB_CM_HEAD_SIZE)) / PAGE_SIZE;
584 636
@@ -601,6 +653,7 @@ void ipoib_cm_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
601 653
602 skb_put_frags(skb, IPOIB_CM_HEAD_SIZE, wc->byte_len, newskb); 654 skb_put_frags(skb, IPOIB_CM_HEAD_SIZE, wc->byte_len, newskb);
603 655
656copied:
604 skb->protocol = ((struct ipoib_header *) skb->data)->proto; 657 skb->protocol = ((struct ipoib_header *) skb->data)->proto;
605 skb_reset_mac_header(skb); 658 skb_reset_mac_header(skb);
606 skb_pull(skb, IPOIB_ENCAP_LEN); 659 skb_pull(skb, IPOIB_ENCAP_LEN);
@@ -620,7 +673,10 @@ repost:
620 ipoib_warn(priv, "ipoib_cm_post_receive_srq failed " 673 ipoib_warn(priv, "ipoib_cm_post_receive_srq failed "
621 "for buf %d\n", wr_id); 674 "for buf %d\n", wr_id);
622 } else { 675 } else {
623 if (unlikely(ipoib_cm_post_receive_nonsrq(dev, p, wr_id))) { 676 if (unlikely(ipoib_cm_post_receive_nonsrq(dev, p,
677 &priv->cm.rx_wr,
678 priv->cm.rx_sge,
679 wr_id))) {
624 --p->recv_count; 680 --p->recv_count;
625 ipoib_warn(priv, "ipoib_cm_post_receive_nonsrq failed " 681 ipoib_warn(priv, "ipoib_cm_post_receive_nonsrq failed "
626 "for buf %d\n", wr_id); 682 "for buf %d\n", wr_id);
@@ -647,7 +703,7 @@ static inline int post_send(struct ipoib_dev_priv *priv,
647void ipoib_cm_send(struct net_device *dev, struct sk_buff *skb, struct ipoib_cm_tx *tx) 703void ipoib_cm_send(struct net_device *dev, struct sk_buff *skb, struct ipoib_cm_tx *tx)
648{ 704{
649 struct ipoib_dev_priv *priv = netdev_priv(dev); 705 struct ipoib_dev_priv *priv = netdev_priv(dev);
650 struct ipoib_tx_buf *tx_req; 706 struct ipoib_cm_tx_buf *tx_req;
651 u64 addr; 707 u64 addr;
652 708
653 if (unlikely(skb->len > tx->mtu)) { 709 if (unlikely(skb->len > tx->mtu)) {
@@ -678,7 +734,7 @@ void ipoib_cm_send(struct net_device *dev, struct sk_buff *skb, struct ipoib_cm_
678 return; 734 return;
679 } 735 }
680 736
681 tx_req->mapping[0] = addr; 737 tx_req->mapping = addr;
682 738
683 if (unlikely(post_send(priv, tx, tx->tx_head & (ipoib_sendq_size - 1), 739 if (unlikely(post_send(priv, tx, tx->tx_head & (ipoib_sendq_size - 1),
684 addr, skb->len))) { 740 addr, skb->len))) {
@@ -703,7 +759,7 @@ void ipoib_cm_handle_tx_wc(struct net_device *dev, struct ib_wc *wc)
703 struct ipoib_dev_priv *priv = netdev_priv(dev); 759 struct ipoib_dev_priv *priv = netdev_priv(dev);
704 struct ipoib_cm_tx *tx = wc->qp->qp_context; 760 struct ipoib_cm_tx *tx = wc->qp->qp_context;
705 unsigned int wr_id = wc->wr_id & ~IPOIB_OP_CM; 761 unsigned int wr_id = wc->wr_id & ~IPOIB_OP_CM;
706 struct ipoib_tx_buf *tx_req; 762 struct ipoib_cm_tx_buf *tx_req;
707 unsigned long flags; 763 unsigned long flags;
708 764
709 ipoib_dbg_data(priv, "cm send completion: id %d, status: %d\n", 765 ipoib_dbg_data(priv, "cm send completion: id %d, status: %d\n",
@@ -717,7 +773,7 @@ void ipoib_cm_handle_tx_wc(struct net_device *dev, struct ib_wc *wc)
717 773
718 tx_req = &tx->tx_ring[wr_id]; 774 tx_req = &tx->tx_ring[wr_id];
719 775
720 ib_dma_unmap_single(priv->ca, tx_req->mapping[0], tx_req->skb->len, DMA_TO_DEVICE); 776 ib_dma_unmap_single(priv->ca, tx_req->mapping, tx_req->skb->len, DMA_TO_DEVICE);
721 777
722 /* FIXME: is this right? Shouldn't we only increment on success? */ 778 /* FIXME: is this right? Shouldn't we only increment on success? */
723 ++dev->stats.tx_packets; 779 ++dev->stats.tx_packets;
@@ -1087,7 +1143,7 @@ err_tx:
1087static void ipoib_cm_tx_destroy(struct ipoib_cm_tx *p) 1143static void ipoib_cm_tx_destroy(struct ipoib_cm_tx *p)
1088{ 1144{
1089 struct ipoib_dev_priv *priv = netdev_priv(p->dev); 1145 struct ipoib_dev_priv *priv = netdev_priv(p->dev);
1090 struct ipoib_tx_buf *tx_req; 1146 struct ipoib_cm_tx_buf *tx_req;
1091 unsigned long flags; 1147 unsigned long flags;
1092 unsigned long begin; 1148 unsigned long begin;
1093 1149
@@ -1115,7 +1171,7 @@ timeout:
1115 1171
1116 while ((int) p->tx_tail - (int) p->tx_head < 0) { 1172 while ((int) p->tx_tail - (int) p->tx_head < 0) {
1117 tx_req = &p->tx_ring[p->tx_tail & (ipoib_sendq_size - 1)]; 1173 tx_req = &p->tx_ring[p->tx_tail & (ipoib_sendq_size - 1)];
1118 ib_dma_unmap_single(priv->ca, tx_req->mapping[0], tx_req->skb->len, 1174 ib_dma_unmap_single(priv->ca, tx_req->mapping, tx_req->skb->len,
1119 DMA_TO_DEVICE); 1175 DMA_TO_DEVICE);
1120 dev_kfree_skb_any(tx_req->skb); 1176 dev_kfree_skb_any(tx_req->skb);
1121 ++p->tx_tail; 1177 ++p->tx_tail;
@@ -1384,7 +1440,9 @@ static ssize_t set_mode(struct device *d, struct device_attribute *attr,
1384 ipoib_warn(priv, "enabling connected mode " 1440 ipoib_warn(priv, "enabling connected mode "
1385 "will cause multicast packet drops\n"); 1441 "will cause multicast packet drops\n");
1386 1442
1443 rtnl_lock();
1387 dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_SG | NETIF_F_TSO); 1444 dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_SG | NETIF_F_TSO);
1445 rtnl_unlock();
1388 priv->tx_wr.send_flags &= ~IB_SEND_IP_CSUM; 1446 priv->tx_wr.send_flags &= ~IB_SEND_IP_CSUM;
1389 1447
1390 ipoib_flush_paths(dev); 1448 ipoib_flush_paths(dev);
@@ -1393,14 +1451,16 @@ static ssize_t set_mode(struct device *d, struct device_attribute *attr,
1393 1451
1394 if (!strcmp(buf, "datagram\n")) { 1452 if (!strcmp(buf, "datagram\n")) {
1395 clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags); 1453 clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
1396 dev->mtu = min(priv->mcast_mtu, dev->mtu);
1397 ipoib_flush_paths(dev);
1398 1454
1455 rtnl_lock();
1399 if (test_bit(IPOIB_FLAG_CSUM, &priv->flags)) { 1456 if (test_bit(IPOIB_FLAG_CSUM, &priv->flags)) {
1400 dev->features |= NETIF_F_IP_CSUM | NETIF_F_SG; 1457 dev->features |= NETIF_F_IP_CSUM | NETIF_F_SG;
1401 if (priv->hca_caps & IB_DEVICE_UD_TSO) 1458 if (priv->hca_caps & IB_DEVICE_UD_TSO)
1402 dev->features |= NETIF_F_TSO; 1459 dev->features |= NETIF_F_TSO;
1403 } 1460 }
1461 dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
1462 rtnl_unlock();
1463 ipoib_flush_paths(dev);
1404 1464
1405 return count; 1465 return count;
1406 } 1466 }
@@ -1485,15 +1545,7 @@ int ipoib_cm_dev_init(struct net_device *dev)
1485 priv->cm.num_frags = IPOIB_CM_RX_SG; 1545 priv->cm.num_frags = IPOIB_CM_RX_SG;
1486 } 1546 }
1487 1547
1488 for (i = 0; i < priv->cm.num_frags; ++i) 1548 ipoib_cm_init_rx_wr(dev, &priv->cm.rx_wr, priv->cm.rx_sge);
1489 priv->cm.rx_sge[i].lkey = priv->mr->lkey;
1490
1491 priv->cm.rx_sge[0].length = IPOIB_CM_HEAD_SIZE;
1492 for (i = 1; i < priv->cm.num_frags; ++i)
1493 priv->cm.rx_sge[i].length = PAGE_SIZE;
1494 priv->cm.rx_wr.next = NULL;
1495 priv->cm.rx_wr.sg_list = priv->cm.rx_sge;
1496 priv->cm.rx_wr.num_sge = priv->cm.num_frags;
1497 1549
1498 if (ipoib_cm_has_srq(dev)) { 1550 if (ipoib_cm_has_srq(dev)) {
1499 for (i = 0; i < ipoib_recvq_size; ++i) { 1551 for (i = 0; i < ipoib_recvq_size; ++i) {
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
index 10279b79c44d..66af5c1a76e5 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ethtool.c
@@ -86,11 +86,57 @@ static int ipoib_set_coalesce(struct net_device *dev,
86 return 0; 86 return 0;
87} 87}
88 88
89static const char ipoib_stats_keys[][ETH_GSTRING_LEN] = {
90 "LRO aggregated", "LRO flushed",
91 "LRO avg aggr", "LRO no desc"
92};
93
94static void ipoib_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
95{
96 switch (stringset) {
97 case ETH_SS_STATS:
98 memcpy(data, *ipoib_stats_keys, sizeof(ipoib_stats_keys));
99 break;
100 }
101}
102
103static int ipoib_get_sset_count(struct net_device *dev, int sset)
104{
105 switch (sset) {
106 case ETH_SS_STATS:
107 return ARRAY_SIZE(ipoib_stats_keys);
108 default:
109 return -EOPNOTSUPP;
110 }
111}
112
113static void ipoib_get_ethtool_stats(struct net_device *dev,
114 struct ethtool_stats *stats, uint64_t *data)
115{
116 struct ipoib_dev_priv *priv = netdev_priv(dev);
117 int index = 0;
118
119 /* Get LRO statistics */
120 data[index++] = priv->lro.lro_mgr.stats.aggregated;
121 data[index++] = priv->lro.lro_mgr.stats.flushed;
122 if (priv->lro.lro_mgr.stats.flushed)
123 data[index++] = priv->lro.lro_mgr.stats.aggregated /
124 priv->lro.lro_mgr.stats.flushed;
125 else
126 data[index++] = 0;
127 data[index++] = priv->lro.lro_mgr.stats.no_desc;
128}
129
89static const struct ethtool_ops ipoib_ethtool_ops = { 130static const struct ethtool_ops ipoib_ethtool_ops = {
90 .get_drvinfo = ipoib_get_drvinfo, 131 .get_drvinfo = ipoib_get_drvinfo,
91 .get_tso = ethtool_op_get_tso, 132 .get_tso = ethtool_op_get_tso,
92 .get_coalesce = ipoib_get_coalesce, 133 .get_coalesce = ipoib_get_coalesce,
93 .set_coalesce = ipoib_set_coalesce, 134 .set_coalesce = ipoib_set_coalesce,
135 .get_flags = ethtool_op_get_flags,
136 .set_flags = ethtool_op_set_flags,
137 .get_strings = ipoib_get_strings,
138 .get_sset_count = ipoib_get_sset_count,
139 .get_ethtool_stats = ipoib_get_ethtool_stats,
94}; 140};
95 141
96void ipoib_set_ethtool_ops(struct net_device *dev) 142void ipoib_set_ethtool_ops(struct net_device *dev)
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_fs.c b/drivers/infiniband/ulp/ipoib/ipoib_fs.c
index 8b882bbd1d05..961c585da216 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_fs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_fs.c
@@ -28,8 +28,6 @@
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE. 30 * SOFTWARE.
31 *
32 * $Id: ipoib_fs.c 1389 2004-12-27 22:56:47Z roland $
33 */ 31 */
34 32
35#include <linux/err.h> 33#include <linux/err.h>
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index f429bce24c20..66cafa20c246 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -31,8 +31,6 @@
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE. 33 * SOFTWARE.
34 *
35 * $Id: ipoib_ib.c 1386 2004-12-27 16:23:17Z roland $
36 */ 34 */
37 35
38#include <linux/delay.h> 36#include <linux/delay.h>
@@ -290,7 +288,10 @@ static void ipoib_ib_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
290 if (test_bit(IPOIB_FLAG_CSUM, &priv->flags) && likely(wc->csum_ok)) 288 if (test_bit(IPOIB_FLAG_CSUM, &priv->flags) && likely(wc->csum_ok))
291 skb->ip_summed = CHECKSUM_UNNECESSARY; 289 skb->ip_summed = CHECKSUM_UNNECESSARY;
292 290
293 netif_receive_skb(skb); 291 if (dev->features & NETIF_F_LRO)
292 lro_receive_skb(&priv->lro.lro_mgr, skb, NULL);
293 else
294 netif_receive_skb(skb);
294 295
295repost: 296repost:
296 if (unlikely(ipoib_ib_post_receive(dev, wr_id))) 297 if (unlikely(ipoib_ib_post_receive(dev, wr_id)))
@@ -442,6 +443,9 @@ poll_more:
442 } 443 }
443 444
444 if (done < budget) { 445 if (done < budget) {
446 if (dev->features & NETIF_F_LRO)
447 lro_flush_all(&priv->lro.lro_mgr);
448
445 netif_rx_complete(dev, napi); 449 netif_rx_complete(dev, napi);
446 if (unlikely(ib_req_notify_cq(priv->recv_cq, 450 if (unlikely(ib_req_notify_cq(priv->recv_cq,
447 IB_CQ_NEXT_COMP | 451 IB_CQ_NEXT_COMP |
@@ -898,7 +902,8 @@ int ipoib_ib_dev_init(struct net_device *dev, struct ib_device *ca, int port)
898 return 0; 902 return 0;
899} 903}
900 904
901static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int pkey_event) 905static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv,
906 enum ipoib_flush_level level)
902{ 907{
903 struct ipoib_dev_priv *cpriv; 908 struct ipoib_dev_priv *cpriv;
904 struct net_device *dev = priv->dev; 909 struct net_device *dev = priv->dev;
@@ -911,7 +916,7 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int pkey_event)
911 * the parent is down. 916 * the parent is down.
912 */ 917 */
913 list_for_each_entry(cpriv, &priv->child_intfs, list) 918 list_for_each_entry(cpriv, &priv->child_intfs, list)
914 __ipoib_ib_dev_flush(cpriv, pkey_event); 919 __ipoib_ib_dev_flush(cpriv, level);
915 920
916 mutex_unlock(&priv->vlan_mutex); 921 mutex_unlock(&priv->vlan_mutex);
917 922
@@ -925,7 +930,7 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int pkey_event)
925 return; 930 return;
926 } 931 }
927 932
928 if (pkey_event) { 933 if (level == IPOIB_FLUSH_HEAVY) {
929 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &new_index)) { 934 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &new_index)) {
930 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); 935 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
931 ipoib_ib_dev_down(dev, 0); 936 ipoib_ib_dev_down(dev, 0);
@@ -943,11 +948,15 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int pkey_event)
943 priv->pkey_index = new_index; 948 priv->pkey_index = new_index;
944 } 949 }
945 950
946 ipoib_dbg(priv, "flushing\n"); 951 if (level == IPOIB_FLUSH_LIGHT) {
952 ipoib_mark_paths_invalid(dev);
953 ipoib_mcast_dev_flush(dev);
954 }
947 955
948 ipoib_ib_dev_down(dev, 0); 956 if (level >= IPOIB_FLUSH_NORMAL)
957 ipoib_ib_dev_down(dev, 0);
949 958
950 if (pkey_event) { 959 if (level == IPOIB_FLUSH_HEAVY) {
951 ipoib_ib_dev_stop(dev, 0); 960 ipoib_ib_dev_stop(dev, 0);
952 ipoib_ib_dev_open(dev); 961 ipoib_ib_dev_open(dev);
953 } 962 }
@@ -957,27 +966,34 @@ static void __ipoib_ib_dev_flush(struct ipoib_dev_priv *priv, int pkey_event)
957 * we get here, don't bring it back up if it's not configured up 966 * we get here, don't bring it back up if it's not configured up
958 */ 967 */
959 if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) { 968 if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
960 ipoib_ib_dev_up(dev); 969 if (level >= IPOIB_FLUSH_NORMAL)
970 ipoib_ib_dev_up(dev);
961 ipoib_mcast_restart_task(&priv->restart_task); 971 ipoib_mcast_restart_task(&priv->restart_task);
962 } 972 }
963} 973}
964 974
965void ipoib_ib_dev_flush(struct work_struct *work) 975void ipoib_ib_dev_flush_light(struct work_struct *work)
976{
977 struct ipoib_dev_priv *priv =
978 container_of(work, struct ipoib_dev_priv, flush_light);
979
980 __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_LIGHT);
981}
982
983void ipoib_ib_dev_flush_normal(struct work_struct *work)
966{ 984{
967 struct ipoib_dev_priv *priv = 985 struct ipoib_dev_priv *priv =
968 container_of(work, struct ipoib_dev_priv, flush_task); 986 container_of(work, struct ipoib_dev_priv, flush_normal);
969 987
970 ipoib_dbg(priv, "Flushing %s\n", priv->dev->name); 988 __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_NORMAL);
971 __ipoib_ib_dev_flush(priv, 0);
972} 989}
973 990
974void ipoib_pkey_event(struct work_struct *work) 991void ipoib_ib_dev_flush_heavy(struct work_struct *work)
975{ 992{
976 struct ipoib_dev_priv *priv = 993 struct ipoib_dev_priv *priv =
977 container_of(work, struct ipoib_dev_priv, pkey_event_task); 994 container_of(work, struct ipoib_dev_priv, flush_heavy);
978 995
979 ipoib_dbg(priv, "Flushing %s and restarting its QP\n", priv->dev->name); 996 __ipoib_ib_dev_flush(priv, IPOIB_FLUSH_HEAVY);
980 __ipoib_ib_dev_flush(priv, 1);
981} 997}
982 998
983void ipoib_ib_dev_cleanup(struct net_device *dev) 999void ipoib_ib_dev_cleanup(struct net_device *dev)
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 2442090ac8d1..8be9ea0436e6 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -30,8 +30,6 @@
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE. 32 * SOFTWARE.
33 *
34 * $Id: ipoib_main.c 1377 2004-12-23 19:57:12Z roland $
35 */ 33 */
36 34
37#include "ipoib.h" 35#include "ipoib.h"
@@ -62,6 +60,15 @@ MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");
62module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444); 60module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);
63MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue"); 61MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");
64 62
63static int lro;
64module_param(lro, bool, 0444);
65MODULE_PARM_DESC(lro, "Enable LRO (Large Receive Offload)");
66
67static int lro_max_aggr = IPOIB_LRO_MAX_AGGR;
68module_param(lro_max_aggr, int, 0644);
69MODULE_PARM_DESC(lro_max_aggr, "LRO: Max packets to be aggregated "
70 "(default = 64)");
71
65#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG 72#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
66int ipoib_debug_level; 73int ipoib_debug_level;
67 74
@@ -350,6 +357,23 @@ void ipoib_path_iter_read(struct ipoib_path_iter *iter,
350 357
351#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */ 358#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */
352 359
360void ipoib_mark_paths_invalid(struct net_device *dev)
361{
362 struct ipoib_dev_priv *priv = netdev_priv(dev);
363 struct ipoib_path *path, *tp;
364
365 spin_lock_irq(&priv->lock);
366
367 list_for_each_entry_safe(path, tp, &priv->path_list, list) {
368 ipoib_dbg(priv, "mark path LID 0x%04x GID " IPOIB_GID_FMT " invalid\n",
369 be16_to_cpu(path->pathrec.dlid),
370 IPOIB_GID_ARG(path->pathrec.dgid));
371 path->valid = 0;
372 }
373
374 spin_unlock_irq(&priv->lock);
375}
376
353void ipoib_flush_paths(struct net_device *dev) 377void ipoib_flush_paths(struct net_device *dev)
354{ 378{
355 struct ipoib_dev_priv *priv = netdev_priv(dev); 379 struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -386,6 +410,7 @@ static void path_rec_completion(int status,
386 struct net_device *dev = path->dev; 410 struct net_device *dev = path->dev;
387 struct ipoib_dev_priv *priv = netdev_priv(dev); 411 struct ipoib_dev_priv *priv = netdev_priv(dev);
388 struct ipoib_ah *ah = NULL; 412 struct ipoib_ah *ah = NULL;
413 struct ipoib_ah *old_ah;
389 struct ipoib_neigh *neigh, *tn; 414 struct ipoib_neigh *neigh, *tn;
390 struct sk_buff_head skqueue; 415 struct sk_buff_head skqueue;
391 struct sk_buff *skb; 416 struct sk_buff *skb;
@@ -409,6 +434,7 @@ static void path_rec_completion(int status,
409 434
410 spin_lock_irqsave(&priv->lock, flags); 435 spin_lock_irqsave(&priv->lock, flags);
411 436
437 old_ah = path->ah;
412 path->ah = ah; 438 path->ah = ah;
413 439
414 if (ah) { 440 if (ah) {
@@ -421,6 +447,17 @@ static void path_rec_completion(int status,
421 __skb_queue_tail(&skqueue, skb); 447 __skb_queue_tail(&skqueue, skb);
422 448
423 list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) { 449 list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {
450 if (neigh->ah) {
451 WARN_ON(neigh->ah != old_ah);
452 /*
453 * Dropping the ah reference inside
454 * priv->lock is safe here, because we
455 * will hold one more reference from
456 * the original value of path->ah (ie
457 * old_ah).
458 */
459 ipoib_put_ah(neigh->ah);
460 }
424 kref_get(&path->ah->ref); 461 kref_get(&path->ah->ref);
425 neigh->ah = path->ah; 462 neigh->ah = path->ah;
426 memcpy(&neigh->dgid.raw, &path->pathrec.dgid.raw, 463 memcpy(&neigh->dgid.raw, &path->pathrec.dgid.raw,
@@ -443,6 +480,7 @@ static void path_rec_completion(int status,
443 while ((skb = __skb_dequeue(&neigh->queue))) 480 while ((skb = __skb_dequeue(&neigh->queue)))
444 __skb_queue_tail(&skqueue, skb); 481 __skb_queue_tail(&skqueue, skb);
445 } 482 }
483 path->valid = 1;
446 } 484 }
447 485
448 path->query = NULL; 486 path->query = NULL;
@@ -450,6 +488,9 @@ static void path_rec_completion(int status,
450 488
451 spin_unlock_irqrestore(&priv->lock, flags); 489 spin_unlock_irqrestore(&priv->lock, flags);
452 490
491 if (old_ah)
492 ipoib_put_ah(old_ah);
493
453 while ((skb = __skb_dequeue(&skqueue))) { 494 while ((skb = __skb_dequeue(&skqueue))) {
454 skb->dev = dev; 495 skb->dev = dev;
455 if (dev_queue_xmit(skb)) 496 if (dev_queue_xmit(skb))
@@ -623,8 +664,9 @@ static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
623 spin_lock(&priv->lock); 664 spin_lock(&priv->lock);
624 665
625 path = __path_find(dev, phdr->hwaddr + 4); 666 path = __path_find(dev, phdr->hwaddr + 4);
626 if (!path) { 667 if (!path || !path->valid) {
627 path = path_rec_create(dev, phdr->hwaddr + 4); 668 if (!path)
669 path = path_rec_create(dev, phdr->hwaddr + 4);
628 if (path) { 670 if (path) {
629 /* put pseudoheader back on for next time */ 671 /* put pseudoheader back on for next time */
630 skb_push(skb, sizeof *phdr); 672 skb_push(skb, sizeof *phdr);
@@ -938,6 +980,54 @@ static const struct header_ops ipoib_header_ops = {
938 .create = ipoib_hard_header, 980 .create = ipoib_hard_header,
939}; 981};
940 982
983static int get_skb_hdr(struct sk_buff *skb, void **iphdr,
984 void **tcph, u64 *hdr_flags, void *priv)
985{
986 unsigned int ip_len;
987 struct iphdr *iph;
988
989 if (unlikely(skb->protocol != htons(ETH_P_IP)))
990 return -1;
991
992 /*
993 * In the future we may add an else clause that verifies the
994 * checksum and allows devices which do not calculate checksum
995 * to use LRO.
996 */
997 if (unlikely(skb->ip_summed != CHECKSUM_UNNECESSARY))
998 return -1;
999
1000 /* Check for non-TCP packet */
1001 skb_reset_network_header(skb);
1002 iph = ip_hdr(skb);
1003 if (iph->protocol != IPPROTO_TCP)
1004 return -1;
1005
1006 ip_len = ip_hdrlen(skb);
1007 skb_set_transport_header(skb, ip_len);
1008 *tcph = tcp_hdr(skb);
1009
1010 /* check if IP header and TCP header are complete */
1011 if (ntohs(iph->tot_len) < ip_len + tcp_hdrlen(skb))
1012 return -1;
1013
1014 *hdr_flags = LRO_IPV4 | LRO_TCP;
1015 *iphdr = iph;
1016
1017 return 0;
1018}
1019
1020static void ipoib_lro_setup(struct ipoib_dev_priv *priv)
1021{
1022 priv->lro.lro_mgr.max_aggr = lro_max_aggr;
1023 priv->lro.lro_mgr.max_desc = IPOIB_MAX_LRO_DESCRIPTORS;
1024 priv->lro.lro_mgr.lro_arr = priv->lro.lro_desc;
1025 priv->lro.lro_mgr.get_skb_header = get_skb_hdr;
1026 priv->lro.lro_mgr.features = LRO_F_NAPI;
1027 priv->lro.lro_mgr.dev = priv->dev;
1028 priv->lro.lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1029}
1030
941static void ipoib_setup(struct net_device *dev) 1031static void ipoib_setup(struct net_device *dev)
942{ 1032{
943 struct ipoib_dev_priv *priv = netdev_priv(dev); 1033 struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -977,10 +1067,11 @@ static void ipoib_setup(struct net_device *dev)
977 1067
978 priv->dev = dev; 1068 priv->dev = dev;
979 1069
1070 ipoib_lro_setup(priv);
1071
980 spin_lock_init(&priv->lock); 1072 spin_lock_init(&priv->lock);
981 spin_lock_init(&priv->tx_lock); 1073 spin_lock_init(&priv->tx_lock);
982 1074
983 mutex_init(&priv->mcast_mutex);
984 mutex_init(&priv->vlan_mutex); 1075 mutex_init(&priv->vlan_mutex);
985 1076
986 INIT_LIST_HEAD(&priv->path_list); 1077 INIT_LIST_HEAD(&priv->path_list);
@@ -989,9 +1080,10 @@ static void ipoib_setup(struct net_device *dev)
989 INIT_LIST_HEAD(&priv->multicast_list); 1080 INIT_LIST_HEAD(&priv->multicast_list);
990 1081
991 INIT_DELAYED_WORK(&priv->pkey_poll_task, ipoib_pkey_poll); 1082 INIT_DELAYED_WORK(&priv->pkey_poll_task, ipoib_pkey_poll);
992 INIT_WORK(&priv->pkey_event_task, ipoib_pkey_event);
993 INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task); 1083 INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task);
994 INIT_WORK(&priv->flush_task, ipoib_ib_dev_flush); 1084 INIT_WORK(&priv->flush_light, ipoib_ib_dev_flush_light);
1085 INIT_WORK(&priv->flush_normal, ipoib_ib_dev_flush_normal);
1086 INIT_WORK(&priv->flush_heavy, ipoib_ib_dev_flush_heavy);
995 INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task); 1087 INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
996 INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah); 1088 INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
997} 1089}
@@ -1154,6 +1246,9 @@ static struct net_device *ipoib_add_port(const char *format,
1154 priv->dev->features |= NETIF_F_SG | NETIF_F_IP_CSUM; 1246 priv->dev->features |= NETIF_F_SG | NETIF_F_IP_CSUM;
1155 } 1247 }
1156 1248
1249 if (lro)
1250 priv->dev->features |= NETIF_F_LRO;
1251
1157 /* 1252 /*
1158 * Set the full membership bit, so that we join the right 1253 * Set the full membership bit, so that we join the right
1159 * broadcast group, etc. 1254 * broadcast group, etc.
@@ -1304,6 +1399,12 @@ static int __init ipoib_init_module(void)
1304 ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP); 1399 ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);
1305#endif 1400#endif
1306 1401
1402 /*
1403 * When copying small received packets, we only copy from the
1404 * linear data part of the SKB, so we rely on this condition.
1405 */
1406 BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);
1407
1307 ret = ipoib_register_debugfs(); 1408 ret = ipoib_register_debugfs();
1308 if (ret) 1409 if (ret)
1309 return ret; 1410 return ret;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index cd2fb955040f..8950e9546f4e 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -30,8 +30,6 @@
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE. 32 * SOFTWARE.
33 *
34 * $Id: ipoib_multicast.c 1362 2004-12-18 15:56:29Z roland $
35 */ 33 */
36 34
37#include <linux/skbuff.h> 35#include <linux/skbuff.h>
@@ -188,6 +186,7 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
188 struct ipoib_dev_priv *priv = netdev_priv(dev); 186 struct ipoib_dev_priv *priv = netdev_priv(dev);
189 struct ipoib_ah *ah; 187 struct ipoib_ah *ah;
190 int ret; 188 int ret;
189 int set_qkey = 0;
191 190
192 mcast->mcmember = *mcmember; 191 mcast->mcmember = *mcmember;
193 192
@@ -202,6 +201,7 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
202 priv->qkey = be32_to_cpu(priv->broadcast->mcmember.qkey); 201 priv->qkey = be32_to_cpu(priv->broadcast->mcmember.qkey);
203 spin_unlock_irq(&priv->lock); 202 spin_unlock_irq(&priv->lock);
204 priv->tx_wr.wr.ud.remote_qkey = priv->qkey; 203 priv->tx_wr.wr.ud.remote_qkey = priv->qkey;
204 set_qkey = 1;
205 } 205 }
206 206
207 if (!test_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags)) { 207 if (!test_bit(IPOIB_MCAST_FLAG_SENDONLY, &mcast->flags)) {
@@ -214,7 +214,7 @@ static int ipoib_mcast_join_finish(struct ipoib_mcast *mcast,
214 } 214 }
215 215
216 ret = ipoib_mcast_attach(dev, be16_to_cpu(mcast->mcmember.mlid), 216 ret = ipoib_mcast_attach(dev, be16_to_cpu(mcast->mcmember.mlid),
217 &mcast->mcmember.mgid); 217 &mcast->mcmember.mgid, set_qkey);
218 if (ret < 0) { 218 if (ret < 0) {
219 ipoib_warn(priv, "couldn't attach QP to multicast group " 219 ipoib_warn(priv, "couldn't attach QP to multicast group "
220 IPOIB_GID_FMT "\n", 220 IPOIB_GID_FMT "\n",
@@ -575,8 +575,11 @@ void ipoib_mcast_join_task(struct work_struct *work)
575 575
576 priv->mcast_mtu = IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu)); 576 priv->mcast_mtu = IPOIB_UD_MTU(ib_mtu_enum_to_int(priv->broadcast->mcmember.mtu));
577 577
578 if (!ipoib_cm_admin_enabled(dev)) 578 if (!ipoib_cm_admin_enabled(dev)) {
579 dev->mtu = min(priv->mcast_mtu, priv->admin_mtu); 579 rtnl_lock();
580 dev_set_mtu(dev, min(priv->mcast_mtu, priv->admin_mtu));
581 rtnl_unlock();
582 }
580 583
581 ipoib_dbg_mcast(priv, "successfully joined all multicast groups\n"); 584 ipoib_dbg_mcast(priv, "successfully joined all multicast groups\n");
582 585
@@ -594,10 +597,6 @@ int ipoib_mcast_start_thread(struct net_device *dev)
594 queue_delayed_work(ipoib_workqueue, &priv->mcast_task, 0); 597 queue_delayed_work(ipoib_workqueue, &priv->mcast_task, 0);
595 mutex_unlock(&mcast_mutex); 598 mutex_unlock(&mcast_mutex);
596 599
597 spin_lock_irq(&priv->lock);
598 set_bit(IPOIB_MCAST_STARTED, &priv->flags);
599 spin_unlock_irq(&priv->lock);
600
601 return 0; 600 return 0;
602} 601}
603 602
@@ -607,10 +606,6 @@ int ipoib_mcast_stop_thread(struct net_device *dev, int flush)
607 606
608 ipoib_dbg_mcast(priv, "stopping multicast thread\n"); 607 ipoib_dbg_mcast(priv, "stopping multicast thread\n");
609 608
610 spin_lock_irq(&priv->lock);
611 clear_bit(IPOIB_MCAST_STARTED, &priv->flags);
612 spin_unlock_irq(&priv->lock);
613
614 mutex_lock(&mcast_mutex); 609 mutex_lock(&mcast_mutex);
615 clear_bit(IPOIB_MCAST_RUN, &priv->flags); 610 clear_bit(IPOIB_MCAST_RUN, &priv->flags);
616 cancel_delayed_work(&priv->mcast_task); 611 cancel_delayed_work(&priv->mcast_task);
@@ -635,10 +630,10 @@ static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
635 IPOIB_GID_ARG(mcast->mcmember.mgid)); 630 IPOIB_GID_ARG(mcast->mcmember.mgid));
636 631
637 /* Remove ourselves from the multicast group */ 632 /* Remove ourselves from the multicast group */
638 ret = ipoib_mcast_detach(dev, be16_to_cpu(mcast->mcmember.mlid), 633 ret = ib_detach_mcast(priv->qp, &mcast->mcmember.mgid,
639 &mcast->mcmember.mgid); 634 be16_to_cpu(mcast->mcmember.mlid));
640 if (ret) 635 if (ret)
641 ipoib_warn(priv, "ipoib_mcast_detach failed (result = %d)\n", ret); 636 ipoib_warn(priv, "ib_detach_mcast failed (result = %d)\n", ret);
642 } 637 }
643 638
644 return 0; 639 return 0;
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
index 8766d29ce3b7..68325119f740 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_verbs.c
@@ -29,24 +29,17 @@
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE. 31 * SOFTWARE.
32 *
33 * $Id: ipoib_verbs.c 1349 2004-12-16 21:09:43Z roland $
34 */ 32 */
35 33
36#include "ipoib.h" 34#include "ipoib.h"
37 35
38int ipoib_mcast_attach(struct net_device *dev, u16 mlid, union ib_gid *mgid) 36int ipoib_mcast_attach(struct net_device *dev, u16 mlid, union ib_gid *mgid, int set_qkey)
39{ 37{
40 struct ipoib_dev_priv *priv = netdev_priv(dev); 38 struct ipoib_dev_priv *priv = netdev_priv(dev);
41 struct ib_qp_attr *qp_attr; 39 struct ib_qp_attr *qp_attr = NULL;
42 int ret; 40 int ret;
43 u16 pkey_index; 41 u16 pkey_index;
44 42
45 ret = -ENOMEM;
46 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
47 if (!qp_attr)
48 goto out;
49
50 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &pkey_index)) { 43 if (ib_find_pkey(priv->ca, priv->port, priv->pkey, &pkey_index)) {
51 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); 44 clear_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
52 ret = -ENXIO; 45 ret = -ENXIO;
@@ -54,18 +47,23 @@ int ipoib_mcast_attach(struct net_device *dev, u16 mlid, union ib_gid *mgid)
54 } 47 }
55 set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags); 48 set_bit(IPOIB_PKEY_ASSIGNED, &priv->flags);
56 49
57 /* set correct QKey for QP */ 50 if (set_qkey) {
58 qp_attr->qkey = priv->qkey; 51 ret = -ENOMEM;
59 ret = ib_modify_qp(priv->qp, qp_attr, IB_QP_QKEY); 52 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL);
60 if (ret) { 53 if (!qp_attr)
61 ipoib_warn(priv, "failed to modify QP, ret = %d\n", ret); 54 goto out;
62 goto out; 55
56 /* set correct QKey for QP */
57 qp_attr->qkey = priv->qkey;
58 ret = ib_modify_qp(priv->qp, qp_attr, IB_QP_QKEY);
59 if (ret) {
60 ipoib_warn(priv, "failed to modify QP, ret = %d\n", ret);
61 goto out;
62 }
63 } 63 }
64 64
65 /* attach QP to multicast group */ 65 /* attach QP to multicast group */
66 mutex_lock(&priv->mcast_mutex);
67 ret = ib_attach_mcast(priv->qp, mgid, mlid); 66 ret = ib_attach_mcast(priv->qp, mgid, mlid);
68 mutex_unlock(&priv->mcast_mutex);
69 if (ret) 67 if (ret)
70 ipoib_warn(priv, "failed to attach to multicast group, ret = %d\n", ret); 68 ipoib_warn(priv, "failed to attach to multicast group, ret = %d\n", ret);
71 69
@@ -74,20 +72,6 @@ out:
74 return ret; 72 return ret;
75} 73}
76 74
77int ipoib_mcast_detach(struct net_device *dev, u16 mlid, union ib_gid *mgid)
78{
79 struct ipoib_dev_priv *priv = netdev_priv(dev);
80 int ret;
81
82 mutex_lock(&priv->mcast_mutex);
83 ret = ib_detach_mcast(priv->qp, mgid, mlid);
84 mutex_unlock(&priv->mcast_mutex);
85 if (ret)
86 ipoib_warn(priv, "ib_detach_mcast failed (result = %d)\n", ret);
87
88 return ret;
89}
90
91int ipoib_init_qp(struct net_device *dev) 75int ipoib_init_qp(struct net_device *dev)
92{ 76{
93 struct ipoib_dev_priv *priv = netdev_priv(dev); 77 struct ipoib_dev_priv *priv = netdev_priv(dev);
@@ -201,7 +185,10 @@ int ipoib_transport_dev_init(struct net_device *dev, struct ib_device *ca)
201 init_attr.recv_cq = priv->recv_cq; 185 init_attr.recv_cq = priv->recv_cq;
202 186
203 if (priv->hca_caps & IB_DEVICE_UD_TSO) 187 if (priv->hca_caps & IB_DEVICE_UD_TSO)
204 init_attr.create_flags = IB_QP_CREATE_IPOIB_UD_LSO; 188 init_attr.create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
189
190 if (priv->hca_caps & IB_DEVICE_BLOCK_MULTICAST_LOOPBACK)
191 init_attr.create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
205 192
206 if (dev->features & NETIF_F_SG) 193 if (dev->features & NETIF_F_SG)
207 init_attr.cap.max_send_sge = MAX_SKB_FRAGS + 1; 194 init_attr.cap.max_send_sge = MAX_SKB_FRAGS + 1;
@@ -289,15 +276,17 @@ void ipoib_event(struct ib_event_handler *handler,
289 if (record->element.port_num != priv->port) 276 if (record->element.port_num != priv->port)
290 return; 277 return;
291 278
292 if (record->event == IB_EVENT_PORT_ERR || 279 ipoib_dbg(priv, "Event %d on device %s port %d\n", record->event,
293 record->event == IB_EVENT_PORT_ACTIVE || 280 record->device->name, record->element.port_num);
294 record->event == IB_EVENT_LID_CHANGE || 281
295 record->event == IB_EVENT_SM_CHANGE || 282 if (record->event == IB_EVENT_SM_CHANGE ||
296 record->event == IB_EVENT_CLIENT_REREGISTER) { 283 record->event == IB_EVENT_CLIENT_REREGISTER) {
297 ipoib_dbg(priv, "Port state change event\n"); 284 queue_work(ipoib_workqueue, &priv->flush_light);
298 queue_work(ipoib_workqueue, &priv->flush_task); 285 } else if (record->event == IB_EVENT_PORT_ERR ||
286 record->event == IB_EVENT_PORT_ACTIVE ||
287 record->event == IB_EVENT_LID_CHANGE) {
288 queue_work(ipoib_workqueue, &priv->flush_normal);
299 } else if (record->event == IB_EVENT_PKEY_CHANGE) { 289 } else if (record->event == IB_EVENT_PKEY_CHANGE) {
300 ipoib_dbg(priv, "P_Key change event on port:%d\n", priv->port); 290 queue_work(ipoib_workqueue, &priv->flush_heavy);
301 queue_work(ipoib_workqueue, &priv->pkey_event_task);
302 } 291 }
303} 292}
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
index 1cdb5cfb0ff1..b08eb56196d3 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_vlan.c
@@ -28,8 +28,6 @@
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE. 30 * SOFTWARE.
31 *
32 * $Id: ipoib_vlan.c 1349 2004-12-16 21:09:43Z roland $
33 */ 31 */
34 32
35#include <linux/module.h> 33#include <linux/module.h>
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index aeb58cae9a3f..5a1cf2580e16 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -42,9 +42,6 @@
42 * Zhenyu Wang 42 * Zhenyu Wang
43 * Modified by: 43 * Modified by:
44 * Erez Zilber 44 * Erez Zilber
45 *
46 *
47 * $Id: iscsi_iser.c 6965 2006-05-07 11:36:20Z ogerlitz $
48 */ 45 */
49 46
50#include <linux/types.h> 47#include <linux/types.h>
@@ -74,6 +71,10 @@
74 71
75#include "iscsi_iser.h" 72#include "iscsi_iser.h"
76 73
74static struct scsi_host_template iscsi_iser_sht;
75static struct iscsi_transport iscsi_iser_transport;
76static struct scsi_transport_template *iscsi_iser_scsi_transport;
77
77static unsigned int iscsi_max_lun = 512; 78static unsigned int iscsi_max_lun = 512;
78module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 79module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
79 80
@@ -94,7 +95,6 @@ iscsi_iser_recv(struct iscsi_conn *conn,
94 struct iscsi_hdr *hdr, char *rx_data, int rx_data_len) 95 struct iscsi_hdr *hdr, char *rx_data, int rx_data_len)
95{ 96{
96 int rc = 0; 97 int rc = 0;
97 uint32_t ret_itt;
98 int datalen; 98 int datalen;
99 int ahslen; 99 int ahslen;
100 100
@@ -110,12 +110,7 @@ iscsi_iser_recv(struct iscsi_conn *conn,
110 /* read AHS */ 110 /* read AHS */
111 ahslen = hdr->hlength * 4; 111 ahslen = hdr->hlength * 4;
112 112
113 /* verify itt (itt encoding: age+cid+itt) */ 113 rc = iscsi_complete_pdu(conn, hdr, rx_data, rx_data_len);
114 rc = iscsi_verify_itt(conn, hdr, &ret_itt);
115
116 if (!rc)
117 rc = iscsi_complete_pdu(conn, hdr, rx_data, rx_data_len);
118
119 if (rc && rc != ISCSI_ERR_NO_SCSI_CMD) 114 if (rc && rc != ISCSI_ERR_NO_SCSI_CMD)
120 goto error; 115 goto error;
121 116
@@ -126,25 +121,33 @@ error:
126 121
127 122
128/** 123/**
129 * iscsi_iser_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands 124 * iscsi_iser_task_init - Initialize task
125 * @task: iscsi task
130 * 126 *
131 **/ 127 * Initialize the task for the scsi command or mgmt command.
128 */
132static int 129static int
133iscsi_iser_cmd_init(struct iscsi_cmd_task *ctask) 130iscsi_iser_task_init(struct iscsi_task *task)
134{ 131{
135 struct iscsi_iser_conn *iser_conn = ctask->conn->dd_data; 132 struct iscsi_iser_conn *iser_conn = task->conn->dd_data;
136 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 133 struct iscsi_iser_task *iser_task = task->dd_data;
137 134
138 iser_ctask->command_sent = 0; 135 /* mgmt task */
139 iser_ctask->iser_conn = iser_conn; 136 if (!task->sc) {
140 iser_ctask_rdma_init(iser_ctask); 137 iser_task->desc.data = task->data;
138 return 0;
139 }
140
141 iser_task->command_sent = 0;
142 iser_task->iser_conn = iser_conn;
143 iser_task_rdma_init(iser_task);
141 return 0; 144 return 0;
142} 145}
143 146
144/** 147/**
145 * iscsi_mtask_xmit - xmit management(immediate) task 148 * iscsi_iser_mtask_xmit - xmit management(immediate) task
146 * @conn: iscsi connection 149 * @conn: iscsi connection
147 * @mtask: task management task 150 * @task: task management task
148 * 151 *
149 * Notes: 152 * Notes:
150 * The function can return -EAGAIN in which case caller must 153 * The function can return -EAGAIN in which case caller must
@@ -153,20 +156,19 @@ iscsi_iser_cmd_init(struct iscsi_cmd_task *ctask)
153 * 156 *
154 **/ 157 **/
155static int 158static int
156iscsi_iser_mtask_xmit(struct iscsi_conn *conn, 159iscsi_iser_mtask_xmit(struct iscsi_conn *conn, struct iscsi_task *task)
157 struct iscsi_mgmt_task *mtask)
158{ 160{
159 int error = 0; 161 int error = 0;
160 162
161 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn->id, mtask->itt); 163 debug_scsi("task deq [cid %d itt 0x%x]\n", conn->id, task->itt);
162 164
163 error = iser_send_control(conn, mtask); 165 error = iser_send_control(conn, task);
164 166
165 /* since iser xmits control with zero copy, mtasks can not be recycled 167 /* since iser xmits control with zero copy, tasks can not be recycled
166 * right after sending them. 168 * right after sending them.
167 * The recycling scheme is based on whether a response is expected 169 * The recycling scheme is based on whether a response is expected
168 * - if yes, the mtask is recycled at iscsi_complete_pdu 170 * - if yes, the task is recycled at iscsi_complete_pdu
169 * - if no, the mtask is recycled at iser_snd_completion 171 * - if no, the task is recycled at iser_snd_completion
170 */ 172 */
171 if (error && error != -ENOBUFS) 173 if (error && error != -ENOBUFS)
172 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 174 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
@@ -175,97 +177,86 @@ iscsi_iser_mtask_xmit(struct iscsi_conn *conn,
175} 177}
176 178
177static int 179static int
178iscsi_iser_ctask_xmit_unsol_data(struct iscsi_conn *conn, 180iscsi_iser_task_xmit_unsol_data(struct iscsi_conn *conn,
179 struct iscsi_cmd_task *ctask) 181 struct iscsi_task *task)
180{ 182{
181 struct iscsi_data hdr; 183 struct iscsi_data hdr;
182 int error = 0; 184 int error = 0;
183 185
184 /* Send data-out PDUs while there's still unsolicited data to send */ 186 /* Send data-out PDUs while there's still unsolicited data to send */
185 while (ctask->unsol_count > 0) { 187 while (task->unsol_count > 0) {
186 iscsi_prep_unsolicit_data_pdu(ctask, &hdr); 188 iscsi_prep_unsolicit_data_pdu(task, &hdr);
187 debug_scsi("Sending data-out: itt 0x%x, data count %d\n", 189 debug_scsi("Sending data-out: itt 0x%x, data count %d\n",
188 hdr.itt, ctask->data_count); 190 hdr.itt, task->data_count);
189 191
190 /* the buffer description has been passed with the command */ 192 /* the buffer description has been passed with the command */
191 /* Send the command */ 193 /* Send the command */
192 error = iser_send_data_out(conn, ctask, &hdr); 194 error = iser_send_data_out(conn, task, &hdr);
193 if (error) { 195 if (error) {
194 ctask->unsol_datasn--; 196 task->unsol_datasn--;
195 goto iscsi_iser_ctask_xmit_unsol_data_exit; 197 goto iscsi_iser_task_xmit_unsol_data_exit;
196 } 198 }
197 ctask->unsol_count -= ctask->data_count; 199 task->unsol_count -= task->data_count;
198 debug_scsi("Need to send %d more as data-out PDUs\n", 200 debug_scsi("Need to send %d more as data-out PDUs\n",
199 ctask->unsol_count); 201 task->unsol_count);
200 } 202 }
201 203
202iscsi_iser_ctask_xmit_unsol_data_exit: 204iscsi_iser_task_xmit_unsol_data_exit:
203 return error; 205 return error;
204} 206}
205 207
206static int 208static int
207iscsi_iser_ctask_xmit(struct iscsi_conn *conn, 209iscsi_iser_task_xmit(struct iscsi_task *task)
208 struct iscsi_cmd_task *ctask)
209{ 210{
210 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 211 struct iscsi_conn *conn = task->conn;
212 struct iscsi_iser_task *iser_task = task->dd_data;
211 int error = 0; 213 int error = 0;
212 214
213 if (ctask->sc->sc_data_direction == DMA_TO_DEVICE) { 215 if (!task->sc)
214 BUG_ON(scsi_bufflen(ctask->sc) == 0); 216 return iscsi_iser_mtask_xmit(conn, task);
217
218 if (task->sc->sc_data_direction == DMA_TO_DEVICE) {
219 BUG_ON(scsi_bufflen(task->sc) == 0);
215 220
216 debug_scsi("cmd [itt %x total %d imm %d unsol_data %d\n", 221 debug_scsi("cmd [itt %x total %d imm %d unsol_data %d\n",
217 ctask->itt, scsi_bufflen(ctask->sc), 222 task->itt, scsi_bufflen(task->sc),
218 ctask->imm_count, ctask->unsol_count); 223 task->imm_count, task->unsol_count);
219 } 224 }
220 225
221 debug_scsi("ctask deq [cid %d itt 0x%x]\n", 226 debug_scsi("task deq [cid %d itt 0x%x]\n",
222 conn->id, ctask->itt); 227 conn->id, task->itt);
223 228
224 /* Send the cmd PDU */ 229 /* Send the cmd PDU */
225 if (!iser_ctask->command_sent) { 230 if (!iser_task->command_sent) {
226 error = iser_send_command(conn, ctask); 231 error = iser_send_command(conn, task);
227 if (error) 232 if (error)
228 goto iscsi_iser_ctask_xmit_exit; 233 goto iscsi_iser_task_xmit_exit;
229 iser_ctask->command_sent = 1; 234 iser_task->command_sent = 1;
230 } 235 }
231 236
232 /* Send unsolicited data-out PDU(s) if necessary */ 237 /* Send unsolicited data-out PDU(s) if necessary */
233 if (ctask->unsol_count) 238 if (task->unsol_count)
234 error = iscsi_iser_ctask_xmit_unsol_data(conn, ctask); 239 error = iscsi_iser_task_xmit_unsol_data(conn, task);
235 240
236 iscsi_iser_ctask_xmit_exit: 241 iscsi_iser_task_xmit_exit:
237 if (error && error != -ENOBUFS) 242 if (error && error != -ENOBUFS)
238 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 243 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
239 return error; 244 return error;
240} 245}
241 246
242static void 247static void
243iscsi_iser_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask) 248iscsi_iser_cleanup_task(struct iscsi_conn *conn, struct iscsi_task *task)
244{ 249{
245 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 250 struct iscsi_iser_task *iser_task = task->dd_data;
246
247 if (iser_ctask->status == ISER_TASK_STATUS_STARTED) {
248 iser_ctask->status = ISER_TASK_STATUS_COMPLETED;
249 iser_ctask_rdma_finalize(iser_ctask);
250 }
251}
252 251
253static struct iser_conn * 252 /* mgmt tasks do not need special cleanup */
254iscsi_iser_ib_conn_lookup(__u64 ep_handle) 253 if (!task->sc)
255{ 254 return;
256 struct iser_conn *ib_conn;
257 struct iser_conn *uib_conn = (struct iser_conn *)(unsigned long)ep_handle;
258 255
259 mutex_lock(&ig.connlist_mutex); 256 if (iser_task->status == ISER_TASK_STATUS_STARTED) {
260 list_for_each_entry(ib_conn, &ig.connlist, conn_list) { 257 iser_task->status = ISER_TASK_STATUS_COMPLETED;
261 if (ib_conn == uib_conn) { 258 iser_task_rdma_finalize(iser_task);
262 mutex_unlock(&ig.connlist_mutex);
263 return ib_conn;
264 }
265 } 259 }
266 mutex_unlock(&ig.connlist_mutex);
267 iser_err("no conn exists for eph %llx\n",(unsigned long long)ep_handle);
268 return NULL;
269} 260}
270 261
271static struct iscsi_cls_conn * 262static struct iscsi_cls_conn *
@@ -275,7 +266,7 @@ iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
275 struct iscsi_cls_conn *cls_conn; 266 struct iscsi_cls_conn *cls_conn;
276 struct iscsi_iser_conn *iser_conn; 267 struct iscsi_iser_conn *iser_conn;
277 268
278 cls_conn = iscsi_conn_setup(cls_session, conn_idx); 269 cls_conn = iscsi_conn_setup(cls_session, sizeof(*iser_conn), conn_idx);
279 if (!cls_conn) 270 if (!cls_conn)
280 return NULL; 271 return NULL;
281 conn = cls_conn->dd_data; 272 conn = cls_conn->dd_data;
@@ -286,21 +277,11 @@ iscsi_iser_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
286 */ 277 */
287 conn->max_recv_dlength = 128; 278 conn->max_recv_dlength = 128;
288 279
289 iser_conn = kzalloc(sizeof(*iser_conn), GFP_KERNEL); 280 iser_conn = conn->dd_data;
290 if (!iser_conn)
291 goto conn_alloc_fail;
292
293 /* currently this is the only field which need to be initiated */
294 rwlock_init(&iser_conn->lock);
295
296 conn->dd_data = iser_conn; 281 conn->dd_data = iser_conn;
297 iser_conn->iscsi_conn = conn; 282 iser_conn->iscsi_conn = conn;
298 283
299 return cls_conn; 284 return cls_conn;
300
301conn_alloc_fail:
302 iscsi_conn_teardown(cls_conn);
303 return NULL;
304} 285}
305 286
306static void 287static void
@@ -308,11 +289,18 @@ iscsi_iser_conn_destroy(struct iscsi_cls_conn *cls_conn)
308{ 289{
309 struct iscsi_conn *conn = cls_conn->dd_data; 290 struct iscsi_conn *conn = cls_conn->dd_data;
310 struct iscsi_iser_conn *iser_conn = conn->dd_data; 291 struct iscsi_iser_conn *iser_conn = conn->dd_data;
292 struct iser_conn *ib_conn = iser_conn->ib_conn;
311 293
312 iscsi_conn_teardown(cls_conn); 294 iscsi_conn_teardown(cls_conn);
313 if (iser_conn->ib_conn) 295 /*
314 iser_conn->ib_conn->iser_conn = NULL; 296 * Userspace will normally call the stop callback and
315 kfree(iser_conn); 297 * already have freed the ib_conn, but if it goofed up then
298 * we free it here.
299 */
300 if (ib_conn) {
301 ib_conn->iser_conn = NULL;
302 iser_conn_put(ib_conn);
303 }
316} 304}
317 305
318static int 306static int
@@ -323,6 +311,7 @@ iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
323 struct iscsi_conn *conn = cls_conn->dd_data; 311 struct iscsi_conn *conn = cls_conn->dd_data;
324 struct iscsi_iser_conn *iser_conn; 312 struct iscsi_iser_conn *iser_conn;
325 struct iser_conn *ib_conn; 313 struct iser_conn *ib_conn;
314 struct iscsi_endpoint *ep;
326 int error; 315 int error;
327 316
328 error = iscsi_conn_bind(cls_session, cls_conn, is_leading); 317 error = iscsi_conn_bind(cls_session, cls_conn, is_leading);
@@ -331,12 +320,14 @@ iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
331 320
332 /* the transport ep handle comes from user space so it must be 321 /* the transport ep handle comes from user space so it must be
333 * verified against the global ib connections list */ 322 * verified against the global ib connections list */
334 ib_conn = iscsi_iser_ib_conn_lookup(transport_eph); 323 ep = iscsi_lookup_endpoint(transport_eph);
335 if (!ib_conn) { 324 if (!ep) {
336 iser_err("can't bind eph %llx\n", 325 iser_err("can't bind eph %llx\n",
337 (unsigned long long)transport_eph); 326 (unsigned long long)transport_eph);
338 return -EINVAL; 327 return -EINVAL;
339 } 328 }
329 ib_conn = ep->dd_data;
330
340 /* binds the iSER connection retrieved from the previously 331 /* binds the iSER connection retrieved from the previously
341 * connected ep_handle to the iSCSI layer connection. exchanges 332 * connected ep_handle to the iSCSI layer connection. exchanges
342 * connection pointers */ 333 * connection pointers */
@@ -344,10 +335,30 @@ iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
344 iser_conn = conn->dd_data; 335 iser_conn = conn->dd_data;
345 ib_conn->iser_conn = iser_conn; 336 ib_conn->iser_conn = iser_conn;
346 iser_conn->ib_conn = ib_conn; 337 iser_conn->ib_conn = ib_conn;
338 iser_conn_get(ib_conn);
339 return 0;
340}
347 341
348 conn->recv_lock = &iser_conn->lock; 342static void
343iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
344{
345 struct iscsi_conn *conn = cls_conn->dd_data;
346 struct iscsi_iser_conn *iser_conn = conn->dd_data;
347 struct iser_conn *ib_conn = iser_conn->ib_conn;
349 348
350 return 0; 349 /*
350 * Userspace may have goofed up and not bound the connection or
351 * might have only partially setup the connection.
352 */
353 if (ib_conn) {
354 iscsi_conn_stop(cls_conn, flag);
355 /*
356 * There is no unbind event so the stop callback
357 * must release the ref from the bind.
358 */
359 iser_conn_put(ib_conn);
360 }
361 iser_conn->ib_conn = NULL;
351} 362}
352 363
353static int 364static int
@@ -363,55 +374,75 @@ iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn)
363 return iscsi_conn_start(cls_conn); 374 return iscsi_conn_start(cls_conn);
364} 375}
365 376
366static struct iscsi_transport iscsi_iser_transport; 377static void iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session)
378{
379 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
380
381 iscsi_host_remove(shost);
382 iscsi_host_free(shost);
383}
367 384
368static struct iscsi_cls_session * 385static struct iscsi_cls_session *
369iscsi_iser_session_create(struct iscsi_transport *iscsit, 386iscsi_iser_session_create(struct iscsi_endpoint *ep,
370 struct scsi_transport_template *scsit, 387 uint16_t cmds_max, uint16_t qdepth,
371 uint16_t cmds_max, uint16_t qdepth, 388 uint32_t initial_cmdsn, uint32_t *hostno)
372 uint32_t initial_cmdsn, uint32_t *hostno)
373{ 389{
374 struct iscsi_cls_session *cls_session; 390 struct iscsi_cls_session *cls_session;
375 struct iscsi_session *session; 391 struct iscsi_session *session;
392 struct Scsi_Host *shost;
376 int i; 393 int i;
377 uint32_t hn; 394 struct iscsi_task *task;
378 struct iscsi_cmd_task *ctask; 395 struct iscsi_iser_task *iser_task;
379 struct iscsi_mgmt_task *mtask; 396 struct iser_conn *ib_conn;
380 struct iscsi_iser_cmd_task *iser_ctask; 397
381 struct iser_desc *desc; 398 shost = iscsi_host_alloc(&iscsi_iser_sht, 0, ISCSI_MAX_CMD_PER_LUN);
399 if (!shost)
400 return NULL;
401 shost->transportt = iscsi_iser_scsi_transport;
402 shost->max_lun = iscsi_max_lun;
403 shost->max_id = 0;
404 shost->max_channel = 0;
405 shost->max_cmd_len = 16;
406
407 /*
408 * older userspace tools (before 2.0-870) did not pass us
409 * the leading conn's ep so this will be NULL;
410 */
411 if (ep)
412 ib_conn = ep->dd_data;
413
414 if (iscsi_host_add(shost,
415 ep ? ib_conn->device->ib_device->dma_device : NULL))
416 goto free_host;
417 *hostno = shost->host_no;
382 418
383 /* 419 /*
384 * we do not support setting can_queue cmd_per_lun from userspace yet 420 * we do not support setting can_queue cmd_per_lun from userspace yet
385 * because we preallocate so many resources 421 * because we preallocate so many resources
386 */ 422 */
387 cls_session = iscsi_session_setup(iscsit, scsit, 423 cls_session = iscsi_session_setup(&iscsi_iser_transport, shost,
388 ISCSI_DEF_XMIT_CMDS_MAX, 424 ISCSI_DEF_XMIT_CMDS_MAX,
389 ISCSI_MAX_CMD_PER_LUN, 425 sizeof(struct iscsi_iser_task),
390 sizeof(struct iscsi_iser_cmd_task), 426 initial_cmdsn, 0);
391 sizeof(struct iser_desc),
392 initial_cmdsn, &hn);
393 if (!cls_session) 427 if (!cls_session)
394 return NULL; 428 goto remove_host;
395 429 session = cls_session->dd_data;
396 *hostno = hn;
397 session = class_to_transport_session(cls_session);
398 430
431 shost->can_queue = session->scsi_cmds_max;
399 /* libiscsi setup itts, data and pool so just set desc fields */ 432 /* libiscsi setup itts, data and pool so just set desc fields */
400 for (i = 0; i < session->cmds_max; i++) { 433 for (i = 0; i < session->cmds_max; i++) {
401 ctask = session->cmds[i]; 434 task = session->cmds[i];
402 iser_ctask = ctask->dd_data; 435 iser_task = task->dd_data;
403 ctask->hdr = (struct iscsi_cmd *)&iser_ctask->desc.iscsi_header; 436 task->hdr = (struct iscsi_cmd *)&iser_task->desc.iscsi_header;
404 ctask->hdr_max = sizeof(iser_ctask->desc.iscsi_header); 437 task->hdr_max = sizeof(iser_task->desc.iscsi_header);
405 }
406
407 for (i = 0; i < session->mgmtpool_max; i++) {
408 mtask = session->mgmt_cmds[i];
409 desc = mtask->dd_data;
410 mtask->hdr = &desc->iscsi_header;
411 desc->data = mtask->data;
412 } 438 }
413
414 return cls_session; 439 return cls_session;
440
441remove_host:
442 iscsi_host_remove(shost);
443free_host:
444 iscsi_host_free(shost);
445 return NULL;
415} 446}
416 447
417static int 448static int
@@ -484,34 +515,37 @@ iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *s
484 stats->custom[3].value = conn->fmr_unalign_cnt; 515 stats->custom[3].value = conn->fmr_unalign_cnt;
485} 516}
486 517
487static int 518static struct iscsi_endpoint *
488iscsi_iser_ep_connect(struct sockaddr *dst_addr, int non_blocking, 519iscsi_iser_ep_connect(struct sockaddr *dst_addr, int non_blocking)
489 __u64 *ep_handle)
490{ 520{
491 int err; 521 int err;
492 struct iser_conn *ib_conn; 522 struct iser_conn *ib_conn;
523 struct iscsi_endpoint *ep;
493 524
494 err = iser_conn_init(&ib_conn); 525 ep = iscsi_create_endpoint(sizeof(*ib_conn));
495 if (err) 526 if (!ep)
496 goto out; 527 return ERR_PTR(-ENOMEM);
497 528
498 err = iser_connect(ib_conn, NULL, (struct sockaddr_in *)dst_addr, non_blocking); 529 ib_conn = ep->dd_data;
499 if (!err) 530 ib_conn->ep = ep;
500 *ep_handle = (__u64)(unsigned long)ib_conn; 531 iser_conn_init(ib_conn);
501 532
502out: 533 err = iser_connect(ib_conn, NULL, (struct sockaddr_in *)dst_addr,
503 return err; 534 non_blocking);
535 if (err) {
536 iscsi_destroy_endpoint(ep);
537 return ERR_PTR(err);
538 }
539 return ep;
504} 540}
505 541
506static int 542static int
507iscsi_iser_ep_poll(__u64 ep_handle, int timeout_ms) 543iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
508{ 544{
509 struct iser_conn *ib_conn = iscsi_iser_ib_conn_lookup(ep_handle); 545 struct iser_conn *ib_conn;
510 int rc; 546 int rc;
511 547
512 if (!ib_conn) 548 ib_conn = ep->dd_data;
513 return -EINVAL;
514
515 rc = wait_event_interruptible_timeout(ib_conn->wait, 549 rc = wait_event_interruptible_timeout(ib_conn->wait,
516 ib_conn->state == ISER_CONN_UP, 550 ib_conn->state == ISER_CONN_UP,
517 msecs_to_jiffies(timeout_ms)); 551 msecs_to_jiffies(timeout_ms));
@@ -533,13 +567,21 @@ iscsi_iser_ep_poll(__u64 ep_handle, int timeout_ms)
533} 567}
534 568
535static void 569static void
536iscsi_iser_ep_disconnect(__u64 ep_handle) 570iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep)
537{ 571{
538 struct iser_conn *ib_conn; 572 struct iser_conn *ib_conn;
539 573
540 ib_conn = iscsi_iser_ib_conn_lookup(ep_handle); 574 ib_conn = ep->dd_data;
541 if (!ib_conn) 575 if (ib_conn->iser_conn)
542 return; 576 /*
577 * Must suspend xmit path if the ep is bound to the
578 * iscsi_conn, so we know we are not accessing the ib_conn
579 * when we free it.
580 *
581 * This may not be bound if the ep poll failed.
582 */
583 iscsi_suspend_tx(ib_conn->iser_conn->iscsi_conn);
584
543 585
544 iser_err("ib conn %p state %d\n",ib_conn, ib_conn->state); 586 iser_err("ib conn %p state %d\n",ib_conn, ib_conn->state);
545 iser_conn_terminate(ib_conn); 587 iser_conn_terminate(ib_conn);
@@ -550,7 +592,6 @@ static struct scsi_host_template iscsi_iser_sht = {
550 .name = "iSCSI Initiator over iSER, v." DRV_VER, 592 .name = "iSCSI Initiator over iSER, v." DRV_VER,
551 .queuecommand = iscsi_queuecommand, 593 .queuecommand = iscsi_queuecommand,
552 .change_queue_depth = iscsi_change_queue_depth, 594 .change_queue_depth = iscsi_change_queue_depth,
553 .can_queue = ISCSI_DEF_XMIT_CMDS_MAX - 1,
554 .sg_tablesize = ISCSI_ISER_SG_TABLESIZE, 595 .sg_tablesize = ISCSI_ISER_SG_TABLESIZE,
555 .max_sectors = 1024, 596 .max_sectors = 1024,
556 .cmd_per_lun = ISCSI_MAX_CMD_PER_LUN, 597 .cmd_per_lun = ISCSI_MAX_CMD_PER_LUN,
@@ -584,17 +625,14 @@ static struct iscsi_transport iscsi_iser_transport = {
584 ISCSI_USERNAME | ISCSI_PASSWORD | 625 ISCSI_USERNAME | ISCSI_PASSWORD |
585 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN | 626 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
586 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO | 627 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
587 ISCSI_PING_TMO | ISCSI_RECV_TMO, 628 ISCSI_PING_TMO | ISCSI_RECV_TMO |
629 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
588 .host_param_mask = ISCSI_HOST_HWADDRESS | 630 .host_param_mask = ISCSI_HOST_HWADDRESS |
589 ISCSI_HOST_NETDEV_NAME | 631 ISCSI_HOST_NETDEV_NAME |
590 ISCSI_HOST_INITIATOR_NAME, 632 ISCSI_HOST_INITIATOR_NAME,
591 .host_template = &iscsi_iser_sht,
592 .conndata_size = sizeof(struct iscsi_conn),
593 .max_lun = ISCSI_ISER_MAX_LUN,
594 .max_cmd_len = ISCSI_ISER_MAX_CMD_LEN,
595 /* session management */ 633 /* session management */
596 .create_session = iscsi_iser_session_create, 634 .create_session = iscsi_iser_session_create,
597 .destroy_session = iscsi_session_teardown, 635 .destroy_session = iscsi_iser_session_destroy,
598 /* connection management */ 636 /* connection management */
599 .create_conn = iscsi_iser_conn_create, 637 .create_conn = iscsi_iser_conn_create,
600 .bind_conn = iscsi_iser_conn_bind, 638 .bind_conn = iscsi_iser_conn_bind,
@@ -603,17 +641,16 @@ static struct iscsi_transport iscsi_iser_transport = {
603 .get_conn_param = iscsi_conn_get_param, 641 .get_conn_param = iscsi_conn_get_param,
604 .get_session_param = iscsi_session_get_param, 642 .get_session_param = iscsi_session_get_param,
605 .start_conn = iscsi_iser_conn_start, 643 .start_conn = iscsi_iser_conn_start,
606 .stop_conn = iscsi_conn_stop, 644 .stop_conn = iscsi_iser_conn_stop,
607 /* iscsi host params */ 645 /* iscsi host params */
608 .get_host_param = iscsi_host_get_param, 646 .get_host_param = iscsi_host_get_param,
609 .set_host_param = iscsi_host_set_param, 647 .set_host_param = iscsi_host_set_param,
610 /* IO */ 648 /* IO */
611 .send_pdu = iscsi_conn_send_pdu, 649 .send_pdu = iscsi_conn_send_pdu,
612 .get_stats = iscsi_iser_conn_get_stats, 650 .get_stats = iscsi_iser_conn_get_stats,
613 .init_cmd_task = iscsi_iser_cmd_init, 651 .init_task = iscsi_iser_task_init,
614 .xmit_cmd_task = iscsi_iser_ctask_xmit, 652 .xmit_task = iscsi_iser_task_xmit,
615 .xmit_mgmt_task = iscsi_iser_mtask_xmit, 653 .cleanup_task = iscsi_iser_cleanup_task,
616 .cleanup_cmd_task = iscsi_iser_cleanup_ctask,
617 /* recovery */ 654 /* recovery */
618 .session_recovery_timedout = iscsi_session_recovery_timedout, 655 .session_recovery_timedout = iscsi_session_recovery_timedout,
619 656
@@ -633,8 +670,6 @@ static int __init iser_init(void)
633 return -EINVAL; 670 return -EINVAL;
634 } 671 }
635 672
636 iscsi_iser_transport.max_lun = iscsi_max_lun;
637
638 memset(&ig, 0, sizeof(struct iser_global)); 673 memset(&ig, 0, sizeof(struct iser_global));
639 674
640 ig.desc_cache = kmem_cache_create("iser_descriptors", 675 ig.desc_cache = kmem_cache_create("iser_descriptors",
@@ -650,7 +685,9 @@ static int __init iser_init(void)
650 mutex_init(&ig.connlist_mutex); 685 mutex_init(&ig.connlist_mutex);
651 INIT_LIST_HEAD(&ig.connlist); 686 INIT_LIST_HEAD(&ig.connlist);
652 687
653 if (!iscsi_register_transport(&iscsi_iser_transport)) { 688 iscsi_iser_scsi_transport = iscsi_register_transport(
689 &iscsi_iser_transport);
690 if (!iscsi_iser_scsi_transport) {
654 iser_err("iscsi_register_transport failed\n"); 691 iser_err("iscsi_register_transport failed\n");
655 err = -EINVAL; 692 err = -EINVAL;
656 goto register_transport_failure; 693 goto register_transport_failure;
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index a8c1b300e34d..81a82628a5f1 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -36,8 +36,6 @@
36 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 36 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
37 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 37 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38 * SOFTWARE. 38 * SOFTWARE.
39 *
40 * $Id: iscsi_iser.h 7051 2006-05-10 12:29:11Z ogerlitz $
41 */ 39 */
42#ifndef __ISCSI_ISER_H__ 40#ifndef __ISCSI_ISER_H__
43#define __ISCSI_ISER_H__ 41#define __ISCSI_ISER_H__
@@ -96,7 +94,6 @@
96 /* support upto 512KB in one RDMA */ 94 /* support upto 512KB in one RDMA */
97#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K) 95#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K)
98#define ISCSI_ISER_MAX_LUN 256 96#define ISCSI_ISER_MAX_LUN 256
99#define ISCSI_ISER_MAX_CMD_LEN 16
100 97
101/* QP settings */ 98/* QP settings */
102/* Maximal bounds on received asynchronous PDUs */ 99/* Maximal bounds on received asynchronous PDUs */
@@ -174,7 +171,8 @@ struct iser_data_buf {
174/* fwd declarations */ 171/* fwd declarations */
175struct iser_device; 172struct iser_device;
176struct iscsi_iser_conn; 173struct iscsi_iser_conn;
177struct iscsi_iser_cmd_task; 174struct iscsi_iser_task;
175struct iscsi_endpoint;
178 176
179struct iser_mem_reg { 177struct iser_mem_reg {
180 u32 lkey; 178 u32 lkey;
@@ -198,7 +196,7 @@ struct iser_regd_buf {
198#define MAX_REGD_BUF_VECTOR_LEN 2 196#define MAX_REGD_BUF_VECTOR_LEN 2
199 197
200struct iser_dto { 198struct iser_dto {
201 struct iscsi_iser_cmd_task *ctask; 199 struct iscsi_iser_task *task;
202 struct iser_conn *ib_conn; 200 struct iser_conn *ib_conn;
203 int notify_enable; 201 int notify_enable;
204 202
@@ -242,7 +240,9 @@ struct iser_device {
242 240
243struct iser_conn { 241struct iser_conn {
244 struct iscsi_iser_conn *iser_conn; /* iser conn for upcalls */ 242 struct iscsi_iser_conn *iser_conn; /* iser conn for upcalls */
243 struct iscsi_endpoint *ep;
245 enum iser_ib_conn_state state; /* rdma connection state */ 244 enum iser_ib_conn_state state; /* rdma connection state */
245 atomic_t refcount;
246 spinlock_t lock; /* used for state changes */ 246 spinlock_t lock; /* used for state changes */
247 struct iser_device *device; /* device context */ 247 struct iser_device *device; /* device context */
248 struct rdma_cm_id *cma_id; /* CMA ID */ 248 struct rdma_cm_id *cma_id; /* CMA ID */
@@ -261,11 +261,9 @@ struct iser_conn {
261struct iscsi_iser_conn { 261struct iscsi_iser_conn {
262 struct iscsi_conn *iscsi_conn;/* ptr to iscsi conn */ 262 struct iscsi_conn *iscsi_conn;/* ptr to iscsi conn */
263 struct iser_conn *ib_conn; /* iSER IB conn */ 263 struct iser_conn *ib_conn; /* iSER IB conn */
264
265 rwlock_t lock;
266}; 264};
267 265
268struct iscsi_iser_cmd_task { 266struct iscsi_iser_task {
269 struct iser_desc desc; 267 struct iser_desc desc;
270 struct iscsi_iser_conn *iser_conn; 268 struct iscsi_iser_conn *iser_conn;
271 enum iser_task_status status; 269 enum iser_task_status status;
@@ -298,22 +296,26 @@ extern int iser_debug_level;
298/* allocate connection resources needed for rdma functionality */ 296/* allocate connection resources needed for rdma functionality */
299int iser_conn_set_full_featured_mode(struct iscsi_conn *conn); 297int iser_conn_set_full_featured_mode(struct iscsi_conn *conn);
300 298
301int iser_send_control(struct iscsi_conn *conn, 299int iser_send_control(struct iscsi_conn *conn,
302 struct iscsi_mgmt_task *mtask); 300 struct iscsi_task *task);
303 301
304int iser_send_command(struct iscsi_conn *conn, 302int iser_send_command(struct iscsi_conn *conn,
305 struct iscsi_cmd_task *ctask); 303 struct iscsi_task *task);
306 304
307int iser_send_data_out(struct iscsi_conn *conn, 305int iser_send_data_out(struct iscsi_conn *conn,
308 struct iscsi_cmd_task *ctask, 306 struct iscsi_task *task,
309 struct iscsi_data *hdr); 307 struct iscsi_data *hdr);
310 308
311void iscsi_iser_recv(struct iscsi_conn *conn, 309void iscsi_iser_recv(struct iscsi_conn *conn,
312 struct iscsi_hdr *hdr, 310 struct iscsi_hdr *hdr,
313 char *rx_data, 311 char *rx_data,
314 int rx_data_len); 312 int rx_data_len);
315 313
316int iser_conn_init(struct iser_conn **ib_conn); 314void iser_conn_init(struct iser_conn *ib_conn);
315
316void iser_conn_get(struct iser_conn *ib_conn);
317
318void iser_conn_put(struct iser_conn *ib_conn);
317 319
318void iser_conn_terminate(struct iser_conn *ib_conn); 320void iser_conn_terminate(struct iser_conn *ib_conn);
319 321
@@ -322,9 +324,9 @@ void iser_rcv_completion(struct iser_desc *desc,
322 324
323void iser_snd_completion(struct iser_desc *desc); 325void iser_snd_completion(struct iser_desc *desc);
324 326
325void iser_ctask_rdma_init(struct iscsi_iser_cmd_task *ctask); 327void iser_task_rdma_init(struct iscsi_iser_task *task);
326 328
327void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task *ctask); 329void iser_task_rdma_finalize(struct iscsi_iser_task *task);
328 330
329void iser_dto_buffs_release(struct iser_dto *dto); 331void iser_dto_buffs_release(struct iser_dto *dto);
330 332
@@ -334,10 +336,10 @@ void iser_reg_single(struct iser_device *device,
334 struct iser_regd_buf *regd_buf, 336 struct iser_regd_buf *regd_buf,
335 enum dma_data_direction direction); 337 enum dma_data_direction direction);
336 338
337void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_cmd_task *ctask, 339void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *task,
338 enum iser_data_dir cmd_dir); 340 enum iser_data_dir cmd_dir);
339 341
340int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *ctask, 342int iser_reg_rdma_mem(struct iscsi_iser_task *task,
341 enum iser_data_dir cmd_dir); 343 enum iser_data_dir cmd_dir);
342 344
343int iser_connect(struct iser_conn *ib_conn, 345int iser_connect(struct iser_conn *ib_conn,
@@ -357,10 +359,10 @@ int iser_post_send(struct iser_desc *tx_desc);
357int iser_conn_state_comp(struct iser_conn *ib_conn, 359int iser_conn_state_comp(struct iser_conn *ib_conn,
358 enum iser_ib_conn_state comp); 360 enum iser_ib_conn_state comp);
359 361
360int iser_dma_map_task_data(struct iscsi_iser_cmd_task *iser_ctask, 362int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
361 struct iser_data_buf *data, 363 struct iser_data_buf *data,
362 enum iser_data_dir iser_dir, 364 enum iser_data_dir iser_dir,
363 enum dma_data_direction dma_dir); 365 enum dma_data_direction dma_dir);
364 366
365void iser_dma_unmap_task_data(struct iscsi_iser_cmd_task *iser_ctask); 367void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task);
366#endif 368#endif
diff --git a/drivers/infiniband/ulp/iser/iser_initiator.c b/drivers/infiniband/ulp/iser/iser_initiator.c
index 08dc81c46f41..cdd283189047 100644
--- a/drivers/infiniband/ulp/iser/iser_initiator.c
+++ b/drivers/infiniband/ulp/iser/iser_initiator.c
@@ -28,8 +28,6 @@
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE. 30 * SOFTWARE.
31 *
32 * $Id: iser_initiator.c 6964 2006-05-07 11:11:43Z ogerlitz $
33 */ 31 */
34#include <linux/kernel.h> 32#include <linux/kernel.h>
35#include <linux/slab.h> 33#include <linux/slab.h>
@@ -66,46 +64,46 @@ static void iser_dto_add_regd_buff(struct iser_dto *dto,
66 64
67/* Register user buffer memory and initialize passive rdma 65/* Register user buffer memory and initialize passive rdma
68 * dto descriptor. Total data size is stored in 66 * dto descriptor. Total data size is stored in
69 * iser_ctask->data[ISER_DIR_IN].data_len 67 * iser_task->data[ISER_DIR_IN].data_len
70 */ 68 */
71static int iser_prepare_read_cmd(struct iscsi_cmd_task *ctask, 69static int iser_prepare_read_cmd(struct iscsi_task *task,
72 unsigned int edtl) 70 unsigned int edtl)
73 71
74{ 72{
75 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 73 struct iscsi_iser_task *iser_task = task->dd_data;
76 struct iser_regd_buf *regd_buf; 74 struct iser_regd_buf *regd_buf;
77 int err; 75 int err;
78 struct iser_hdr *hdr = &iser_ctask->desc.iser_header; 76 struct iser_hdr *hdr = &iser_task->desc.iser_header;
79 struct iser_data_buf *buf_in = &iser_ctask->data[ISER_DIR_IN]; 77 struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN];
80 78
81 err = iser_dma_map_task_data(iser_ctask, 79 err = iser_dma_map_task_data(iser_task,
82 buf_in, 80 buf_in,
83 ISER_DIR_IN, 81 ISER_DIR_IN,
84 DMA_FROM_DEVICE); 82 DMA_FROM_DEVICE);
85 if (err) 83 if (err)
86 return err; 84 return err;
87 85
88 if (edtl > iser_ctask->data[ISER_DIR_IN].data_len) { 86 if (edtl > iser_task->data[ISER_DIR_IN].data_len) {
89 iser_err("Total data length: %ld, less than EDTL: " 87 iser_err("Total data length: %ld, less than EDTL: "
90 "%d, in READ cmd BHS itt: %d, conn: 0x%p\n", 88 "%d, in READ cmd BHS itt: %d, conn: 0x%p\n",
91 iser_ctask->data[ISER_DIR_IN].data_len, edtl, 89 iser_task->data[ISER_DIR_IN].data_len, edtl,
92 ctask->itt, iser_ctask->iser_conn); 90 task->itt, iser_task->iser_conn);
93 return -EINVAL; 91 return -EINVAL;
94 } 92 }
95 93
96 err = iser_reg_rdma_mem(iser_ctask,ISER_DIR_IN); 94 err = iser_reg_rdma_mem(iser_task,ISER_DIR_IN);
97 if (err) { 95 if (err) {
98 iser_err("Failed to set up Data-IN RDMA\n"); 96 iser_err("Failed to set up Data-IN RDMA\n");
99 return err; 97 return err;
100 } 98 }
101 regd_buf = &iser_ctask->rdma_regd[ISER_DIR_IN]; 99 regd_buf = &iser_task->rdma_regd[ISER_DIR_IN];
102 100
103 hdr->flags |= ISER_RSV; 101 hdr->flags |= ISER_RSV;
104 hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey); 102 hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey);
105 hdr->read_va = cpu_to_be64(regd_buf->reg.va); 103 hdr->read_va = cpu_to_be64(regd_buf->reg.va);
106 104
107 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n", 105 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
108 ctask->itt, regd_buf->reg.rkey, 106 task->itt, regd_buf->reg.rkey,
109 (unsigned long long)regd_buf->reg.va); 107 (unsigned long long)regd_buf->reg.va);
110 108
111 return 0; 109 return 0;
@@ -113,43 +111,43 @@ static int iser_prepare_read_cmd(struct iscsi_cmd_task *ctask,
113 111
114/* Register user buffer memory and initialize passive rdma 112/* Register user buffer memory and initialize passive rdma
115 * dto descriptor. Total data size is stored in 113 * dto descriptor. Total data size is stored in
116 * ctask->data[ISER_DIR_OUT].data_len 114 * task->data[ISER_DIR_OUT].data_len
117 */ 115 */
118static int 116static int
119iser_prepare_write_cmd(struct iscsi_cmd_task *ctask, 117iser_prepare_write_cmd(struct iscsi_task *task,
120 unsigned int imm_sz, 118 unsigned int imm_sz,
121 unsigned int unsol_sz, 119 unsigned int unsol_sz,
122 unsigned int edtl) 120 unsigned int edtl)
123{ 121{
124 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 122 struct iscsi_iser_task *iser_task = task->dd_data;
125 struct iser_regd_buf *regd_buf; 123 struct iser_regd_buf *regd_buf;
126 int err; 124 int err;
127 struct iser_dto *send_dto = &iser_ctask->desc.dto; 125 struct iser_dto *send_dto = &iser_task->desc.dto;
128 struct iser_hdr *hdr = &iser_ctask->desc.iser_header; 126 struct iser_hdr *hdr = &iser_task->desc.iser_header;
129 struct iser_data_buf *buf_out = &iser_ctask->data[ISER_DIR_OUT]; 127 struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT];
130 128
131 err = iser_dma_map_task_data(iser_ctask, 129 err = iser_dma_map_task_data(iser_task,
132 buf_out, 130 buf_out,
133 ISER_DIR_OUT, 131 ISER_DIR_OUT,
134 DMA_TO_DEVICE); 132 DMA_TO_DEVICE);
135 if (err) 133 if (err)
136 return err; 134 return err;
137 135
138 if (edtl > iser_ctask->data[ISER_DIR_OUT].data_len) { 136 if (edtl > iser_task->data[ISER_DIR_OUT].data_len) {
139 iser_err("Total data length: %ld, less than EDTL: %d, " 137 iser_err("Total data length: %ld, less than EDTL: %d, "
140 "in WRITE cmd BHS itt: %d, conn: 0x%p\n", 138 "in WRITE cmd BHS itt: %d, conn: 0x%p\n",
141 iser_ctask->data[ISER_DIR_OUT].data_len, 139 iser_task->data[ISER_DIR_OUT].data_len,
142 edtl, ctask->itt, ctask->conn); 140 edtl, task->itt, task->conn);
143 return -EINVAL; 141 return -EINVAL;
144 } 142 }
145 143
146 err = iser_reg_rdma_mem(iser_ctask,ISER_DIR_OUT); 144 err = iser_reg_rdma_mem(iser_task,ISER_DIR_OUT);
147 if (err != 0) { 145 if (err != 0) {
148 iser_err("Failed to register write cmd RDMA mem\n"); 146 iser_err("Failed to register write cmd RDMA mem\n");
149 return err; 147 return err;
150 } 148 }
151 149
152 regd_buf = &iser_ctask->rdma_regd[ISER_DIR_OUT]; 150 regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT];
153 151
154 if (unsol_sz < edtl) { 152 if (unsol_sz < edtl) {
155 hdr->flags |= ISER_WSV; 153 hdr->flags |= ISER_WSV;
@@ -158,13 +156,13 @@ iser_prepare_write_cmd(struct iscsi_cmd_task *ctask,
158 156
159 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X " 157 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
160 "VA:%#llX + unsol:%d\n", 158 "VA:%#llX + unsol:%d\n",
161 ctask->itt, regd_buf->reg.rkey, 159 task->itt, regd_buf->reg.rkey,
162 (unsigned long long)regd_buf->reg.va, unsol_sz); 160 (unsigned long long)regd_buf->reg.va, unsol_sz);
163 } 161 }
164 162
165 if (imm_sz > 0) { 163 if (imm_sz > 0) {
166 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", 164 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
167 ctask->itt, imm_sz); 165 task->itt, imm_sz);
168 iser_dto_add_regd_buff(send_dto, 166 iser_dto_add_regd_buff(send_dto,
169 regd_buf, 167 regd_buf,
170 0, 168 0,
@@ -316,38 +314,38 @@ iser_check_xmit(struct iscsi_conn *conn, void *task)
316/** 314/**
317 * iser_send_command - send command PDU 315 * iser_send_command - send command PDU
318 */ 316 */
319int iser_send_command(struct iscsi_conn *conn, 317int iser_send_command(struct iscsi_conn *conn,
320 struct iscsi_cmd_task *ctask) 318 struct iscsi_task *task)
321{ 319{
322 struct iscsi_iser_conn *iser_conn = conn->dd_data; 320 struct iscsi_iser_conn *iser_conn = conn->dd_data;
323 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 321 struct iscsi_iser_task *iser_task = task->dd_data;
324 struct iser_dto *send_dto = NULL; 322 struct iser_dto *send_dto = NULL;
325 unsigned long edtl; 323 unsigned long edtl;
326 int err = 0; 324 int err = 0;
327 struct iser_data_buf *data_buf; 325 struct iser_data_buf *data_buf;
328 326
329 struct iscsi_cmd *hdr = ctask->hdr; 327 struct iscsi_cmd *hdr = task->hdr;
330 struct scsi_cmnd *sc = ctask->sc; 328 struct scsi_cmnd *sc = task->sc;
331 329
332 if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) { 330 if (!iser_conn_state_comp(iser_conn->ib_conn, ISER_CONN_UP)) {
333 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn); 331 iser_err("Failed to send, conn: 0x%p is not up\n", iser_conn->ib_conn);
334 return -EPERM; 332 return -EPERM;
335 } 333 }
336 if (iser_check_xmit(conn, ctask)) 334 if (iser_check_xmit(conn, task))
337 return -ENOBUFS; 335 return -ENOBUFS;
338 336
339 edtl = ntohl(hdr->data_length); 337 edtl = ntohl(hdr->data_length);
340 338
341 /* build the tx desc regd header and add it to the tx desc dto */ 339 /* build the tx desc regd header and add it to the tx desc dto */
342 iser_ctask->desc.type = ISCSI_TX_SCSI_COMMAND; 340 iser_task->desc.type = ISCSI_TX_SCSI_COMMAND;
343 send_dto = &iser_ctask->desc.dto; 341 send_dto = &iser_task->desc.dto;
344 send_dto->ctask = iser_ctask; 342 send_dto->task = iser_task;
345 iser_create_send_desc(iser_conn, &iser_ctask->desc); 343 iser_create_send_desc(iser_conn, &iser_task->desc);
346 344
347 if (hdr->flags & ISCSI_FLAG_CMD_READ) 345 if (hdr->flags & ISCSI_FLAG_CMD_READ)
348 data_buf = &iser_ctask->data[ISER_DIR_IN]; 346 data_buf = &iser_task->data[ISER_DIR_IN];
349 else 347 else
350 data_buf = &iser_ctask->data[ISER_DIR_OUT]; 348 data_buf = &iser_task->data[ISER_DIR_OUT];
351 349
352 if (scsi_sg_count(sc)) { /* using a scatter list */ 350 if (scsi_sg_count(sc)) { /* using a scatter list */
353 data_buf->buf = scsi_sglist(sc); 351 data_buf->buf = scsi_sglist(sc);
@@ -357,15 +355,15 @@ int iser_send_command(struct iscsi_conn *conn,
357 data_buf->data_len = scsi_bufflen(sc); 355 data_buf->data_len = scsi_bufflen(sc);
358 356
359 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 357 if (hdr->flags & ISCSI_FLAG_CMD_READ) {
360 err = iser_prepare_read_cmd(ctask, edtl); 358 err = iser_prepare_read_cmd(task, edtl);
361 if (err) 359 if (err)
362 goto send_command_error; 360 goto send_command_error;
363 } 361 }
364 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { 362 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) {
365 err = iser_prepare_write_cmd(ctask, 363 err = iser_prepare_write_cmd(task,
366 ctask->imm_count, 364 task->imm_count,
367 ctask->imm_count + 365 task->imm_count +
368 ctask->unsol_count, 366 task->unsol_count,
369 edtl); 367 edtl);
370 if (err) 368 if (err)
371 goto send_command_error; 369 goto send_command_error;
@@ -380,27 +378,27 @@ int iser_send_command(struct iscsi_conn *conn,
380 goto send_command_error; 378 goto send_command_error;
381 } 379 }
382 380
383 iser_ctask->status = ISER_TASK_STATUS_STARTED; 381 iser_task->status = ISER_TASK_STATUS_STARTED;
384 382
385 err = iser_post_send(&iser_ctask->desc); 383 err = iser_post_send(&iser_task->desc);
386 if (!err) 384 if (!err)
387 return 0; 385 return 0;
388 386
389send_command_error: 387send_command_error:
390 iser_dto_buffs_release(send_dto); 388 iser_dto_buffs_release(send_dto);
391 iser_err("conn %p failed ctask->itt %d err %d\n",conn, ctask->itt, err); 389 iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err);
392 return err; 390 return err;
393} 391}
394 392
395/** 393/**
396 * iser_send_data_out - send data out PDU 394 * iser_send_data_out - send data out PDU
397 */ 395 */
398int iser_send_data_out(struct iscsi_conn *conn, 396int iser_send_data_out(struct iscsi_conn *conn,
399 struct iscsi_cmd_task *ctask, 397 struct iscsi_task *task,
400 struct iscsi_data *hdr) 398 struct iscsi_data *hdr)
401{ 399{
402 struct iscsi_iser_conn *iser_conn = conn->dd_data; 400 struct iscsi_iser_conn *iser_conn = conn->dd_data;
403 struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data; 401 struct iscsi_iser_task *iser_task = task->dd_data;
404 struct iser_desc *tx_desc = NULL; 402 struct iser_desc *tx_desc = NULL;
405 struct iser_dto *send_dto = NULL; 403 struct iser_dto *send_dto = NULL;
406 unsigned long buf_offset; 404 unsigned long buf_offset;
@@ -413,7 +411,7 @@ int iser_send_data_out(struct iscsi_conn *conn,
413 return -EPERM; 411 return -EPERM;
414 } 412 }
415 413
416 if (iser_check_xmit(conn, ctask)) 414 if (iser_check_xmit(conn, task))
417 return -ENOBUFS; 415 return -ENOBUFS;
418 416
419 itt = (__force uint32_t)hdr->itt; 417 itt = (__force uint32_t)hdr->itt;
@@ -434,7 +432,7 @@ int iser_send_data_out(struct iscsi_conn *conn,
434 432
435 /* build the tx desc regd header and add it to the tx desc dto */ 433 /* build the tx desc regd header and add it to the tx desc dto */
436 send_dto = &tx_desc->dto; 434 send_dto = &tx_desc->dto;
437 send_dto->ctask = iser_ctask; 435 send_dto->task = iser_task;
438 iser_create_send_desc(iser_conn, tx_desc); 436 iser_create_send_desc(iser_conn, tx_desc);
439 437
440 iser_reg_single(iser_conn->ib_conn->device, 438 iser_reg_single(iser_conn->ib_conn->device,
@@ -442,15 +440,15 @@ int iser_send_data_out(struct iscsi_conn *conn,
442 440
443 /* all data was registered for RDMA, we can use the lkey */ 441 /* all data was registered for RDMA, we can use the lkey */
444 iser_dto_add_regd_buff(send_dto, 442 iser_dto_add_regd_buff(send_dto,
445 &iser_ctask->rdma_regd[ISER_DIR_OUT], 443 &iser_task->rdma_regd[ISER_DIR_OUT],
446 buf_offset, 444 buf_offset,
447 data_seg_len); 445 data_seg_len);
448 446
449 if (buf_offset + data_seg_len > iser_ctask->data[ISER_DIR_OUT].data_len) { 447 if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) {
450 iser_err("Offset:%ld & DSL:%ld in Data-Out " 448 iser_err("Offset:%ld & DSL:%ld in Data-Out "
451 "inconsistent with total len:%ld, itt:%d\n", 449 "inconsistent with total len:%ld, itt:%d\n",
452 buf_offset, data_seg_len, 450 buf_offset, data_seg_len,
453 iser_ctask->data[ISER_DIR_OUT].data_len, itt); 451 iser_task->data[ISER_DIR_OUT].data_len, itt);
454 err = -EINVAL; 452 err = -EINVAL;
455 goto send_data_out_error; 453 goto send_data_out_error;
456 } 454 }
@@ -470,10 +468,11 @@ send_data_out_error:
470} 468}
471 469
472int iser_send_control(struct iscsi_conn *conn, 470int iser_send_control(struct iscsi_conn *conn,
473 struct iscsi_mgmt_task *mtask) 471 struct iscsi_task *task)
474{ 472{
475 struct iscsi_iser_conn *iser_conn = conn->dd_data; 473 struct iscsi_iser_conn *iser_conn = conn->dd_data;
476 struct iser_desc *mdesc = mtask->dd_data; 474 struct iscsi_iser_task *iser_task = task->dd_data;
475 struct iser_desc *mdesc = &iser_task->desc;
477 struct iser_dto *send_dto = NULL; 476 struct iser_dto *send_dto = NULL;
478 unsigned long data_seg_len; 477 unsigned long data_seg_len;
479 int err = 0; 478 int err = 0;
@@ -485,27 +484,27 @@ int iser_send_control(struct iscsi_conn *conn,
485 return -EPERM; 484 return -EPERM;
486 } 485 }
487 486
488 if (iser_check_xmit(conn,mtask)) 487 if (iser_check_xmit(conn, task))
489 return -ENOBUFS; 488 return -ENOBUFS;
490 489
491 /* build the tx desc regd header and add it to the tx desc dto */ 490 /* build the tx desc regd header and add it to the tx desc dto */
492 mdesc->type = ISCSI_TX_CONTROL; 491 mdesc->type = ISCSI_TX_CONTROL;
493 send_dto = &mdesc->dto; 492 send_dto = &mdesc->dto;
494 send_dto->ctask = NULL; 493 send_dto->task = NULL;
495 iser_create_send_desc(iser_conn, mdesc); 494 iser_create_send_desc(iser_conn, mdesc);
496 495
497 device = iser_conn->ib_conn->device; 496 device = iser_conn->ib_conn->device;
498 497
499 iser_reg_single(device, send_dto->regd[0], DMA_TO_DEVICE); 498 iser_reg_single(device, send_dto->regd[0], DMA_TO_DEVICE);
500 499
501 data_seg_len = ntoh24(mtask->hdr->dlength); 500 data_seg_len = ntoh24(task->hdr->dlength);
502 501
503 if (data_seg_len > 0) { 502 if (data_seg_len > 0) {
504 regd_buf = &mdesc->data_regd_buf; 503 regd_buf = &mdesc->data_regd_buf;
505 memset(regd_buf, 0, sizeof(struct iser_regd_buf)); 504 memset(regd_buf, 0, sizeof(struct iser_regd_buf));
506 regd_buf->device = device; 505 regd_buf->device = device;
507 regd_buf->virt_addr = mtask->data; 506 regd_buf->virt_addr = task->data;
508 regd_buf->data_size = mtask->data_count; 507 regd_buf->data_size = task->data_count;
509 iser_reg_single(device, regd_buf, 508 iser_reg_single(device, regd_buf,
510 DMA_TO_DEVICE); 509 DMA_TO_DEVICE);
511 iser_dto_add_regd_buff(send_dto, regd_buf, 510 iser_dto_add_regd_buff(send_dto, regd_buf,
@@ -535,15 +534,13 @@ send_control_error:
535void iser_rcv_completion(struct iser_desc *rx_desc, 534void iser_rcv_completion(struct iser_desc *rx_desc,
536 unsigned long dto_xfer_len) 535 unsigned long dto_xfer_len)
537{ 536{
538 struct iser_dto *dto = &rx_desc->dto; 537 struct iser_dto *dto = &rx_desc->dto;
539 struct iscsi_iser_conn *conn = dto->ib_conn->iser_conn; 538 struct iscsi_iser_conn *conn = dto->ib_conn->iser_conn;
540 struct iscsi_session *session = conn->iscsi_conn->session; 539 struct iscsi_task *task;
541 struct iscsi_cmd_task *ctask; 540 struct iscsi_iser_task *iser_task;
542 struct iscsi_iser_cmd_task *iser_ctask;
543 struct iscsi_hdr *hdr; 541 struct iscsi_hdr *hdr;
544 char *rx_data = NULL; 542 char *rx_data = NULL;
545 int rx_data_len = 0; 543 int rx_data_len = 0;
546 unsigned int itt;
547 unsigned char opcode; 544 unsigned char opcode;
548 545
549 hdr = &rx_desc->iscsi_header; 546 hdr = &rx_desc->iscsi_header;
@@ -559,19 +556,24 @@ void iser_rcv_completion(struct iser_desc *rx_desc,
559 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 556 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
560 557
561 if (opcode == ISCSI_OP_SCSI_CMD_RSP) { 558 if (opcode == ISCSI_OP_SCSI_CMD_RSP) {
562 itt = get_itt(hdr->itt); /* mask out cid and age bits */ 559 spin_lock(&conn->iscsi_conn->session->lock);
563 if (!(itt < session->cmds_max)) 560 task = iscsi_itt_to_ctask(conn->iscsi_conn, hdr->itt);
561 if (task)
562 __iscsi_get_task(task);
563 spin_unlock(&conn->iscsi_conn->session->lock);
564
565 if (!task)
564 iser_err("itt can't be matched to task!!! " 566 iser_err("itt can't be matched to task!!! "
565 "conn %p opcode %d cmds_max %d itt %d\n", 567 "conn %p opcode %d itt %d\n",
566 conn->iscsi_conn,opcode,session->cmds_max,itt); 568 conn->iscsi_conn, opcode, hdr->itt);
567 /* use the mapping given with the cmds array indexed by itt */ 569 else {
568 ctask = (struct iscsi_cmd_task *)session->cmds[itt]; 570 iser_task = task->dd_data;
569 iser_ctask = ctask->dd_data; 571 iser_dbg("itt %d task %p\n",hdr->itt, task);
570 iser_dbg("itt %d ctask %p\n",itt,ctask); 572 iser_task->status = ISER_TASK_STATUS_COMPLETED;
571 iser_ctask->status = ISER_TASK_STATUS_COMPLETED; 573 iser_task_rdma_finalize(iser_task);
572 iser_ctask_rdma_finalize(iser_ctask); 574 iscsi_put_task(task);
575 }
573 } 576 }
574
575 iser_dto_buffs_release(dto); 577 iser_dto_buffs_release(dto);
576 578
577 iscsi_iser_recv(conn->iscsi_conn, hdr, rx_data, rx_data_len); 579 iscsi_iser_recv(conn->iscsi_conn, hdr, rx_data, rx_data_len);
@@ -592,7 +594,7 @@ void iser_snd_completion(struct iser_desc *tx_desc)
592 struct iser_conn *ib_conn = dto->ib_conn; 594 struct iser_conn *ib_conn = dto->ib_conn;
593 struct iscsi_iser_conn *iser_conn = ib_conn->iser_conn; 595 struct iscsi_iser_conn *iser_conn = ib_conn->iser_conn;
594 struct iscsi_conn *conn = iser_conn->iscsi_conn; 596 struct iscsi_conn *conn = iser_conn->iscsi_conn;
595 struct iscsi_mgmt_task *mtask; 597 struct iscsi_task *task;
596 int resume_tx = 0; 598 int resume_tx = 0;
597 599
598 iser_dbg("Initiator, Data sent dto=0x%p\n", dto); 600 iser_dbg("Initiator, Data sent dto=0x%p\n", dto);
@@ -615,36 +617,31 @@ void iser_snd_completion(struct iser_desc *tx_desc)
615 617
616 if (tx_desc->type == ISCSI_TX_CONTROL) { 618 if (tx_desc->type == ISCSI_TX_CONTROL) {
617 /* this arithmetic is legal by libiscsi dd_data allocation */ 619 /* this arithmetic is legal by libiscsi dd_data allocation */
618 mtask = (void *) ((long)(void *)tx_desc - 620 task = (void *) ((long)(void *)tx_desc -
619 sizeof(struct iscsi_mgmt_task)); 621 sizeof(struct iscsi_task));
620 if (mtask->hdr->itt == RESERVED_ITT) { 622 if (task->hdr->itt == RESERVED_ITT)
621 struct iscsi_session *session = conn->session; 623 iscsi_put_task(task);
622
623 spin_lock(&conn->session->lock);
624 iscsi_free_mgmt_task(conn, mtask);
625 spin_unlock(&session->lock);
626 }
627 } 624 }
628} 625}
629 626
630void iser_ctask_rdma_init(struct iscsi_iser_cmd_task *iser_ctask) 627void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
631 628
632{ 629{
633 iser_ctask->status = ISER_TASK_STATUS_INIT; 630 iser_task->status = ISER_TASK_STATUS_INIT;
634 631
635 iser_ctask->dir[ISER_DIR_IN] = 0; 632 iser_task->dir[ISER_DIR_IN] = 0;
636 iser_ctask->dir[ISER_DIR_OUT] = 0; 633 iser_task->dir[ISER_DIR_OUT] = 0;
637 634
638 iser_ctask->data[ISER_DIR_IN].data_len = 0; 635 iser_task->data[ISER_DIR_IN].data_len = 0;
639 iser_ctask->data[ISER_DIR_OUT].data_len = 0; 636 iser_task->data[ISER_DIR_OUT].data_len = 0;
640 637
641 memset(&iser_ctask->rdma_regd[ISER_DIR_IN], 0, 638 memset(&iser_task->rdma_regd[ISER_DIR_IN], 0,
642 sizeof(struct iser_regd_buf)); 639 sizeof(struct iser_regd_buf));
643 memset(&iser_ctask->rdma_regd[ISER_DIR_OUT], 0, 640 memset(&iser_task->rdma_regd[ISER_DIR_OUT], 0,
644 sizeof(struct iser_regd_buf)); 641 sizeof(struct iser_regd_buf));
645} 642}
646 643
647void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task *iser_ctask) 644void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
648{ 645{
649 int deferred; 646 int deferred;
650 int is_rdma_aligned = 1; 647 int is_rdma_aligned = 1;
@@ -653,17 +650,17 @@ void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task *iser_ctask)
653 /* if we were reading, copy back to unaligned sglist, 650 /* if we were reading, copy back to unaligned sglist,
654 * anyway dma_unmap and free the copy 651 * anyway dma_unmap and free the copy
655 */ 652 */
656 if (iser_ctask->data_copy[ISER_DIR_IN].copy_buf != NULL) { 653 if (iser_task->data_copy[ISER_DIR_IN].copy_buf != NULL) {
657 is_rdma_aligned = 0; 654 is_rdma_aligned = 0;
658 iser_finalize_rdma_unaligned_sg(iser_ctask, ISER_DIR_IN); 655 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_IN);
659 } 656 }
660 if (iser_ctask->data_copy[ISER_DIR_OUT].copy_buf != NULL) { 657 if (iser_task->data_copy[ISER_DIR_OUT].copy_buf != NULL) {
661 is_rdma_aligned = 0; 658 is_rdma_aligned = 0;
662 iser_finalize_rdma_unaligned_sg(iser_ctask, ISER_DIR_OUT); 659 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT);
663 } 660 }
664 661
665 if (iser_ctask->dir[ISER_DIR_IN]) { 662 if (iser_task->dir[ISER_DIR_IN]) {
666 regd = &iser_ctask->rdma_regd[ISER_DIR_IN]; 663 regd = &iser_task->rdma_regd[ISER_DIR_IN];
667 deferred = iser_regd_buff_release(regd); 664 deferred = iser_regd_buff_release(regd);
668 if (deferred) { 665 if (deferred) {
669 iser_err("%d references remain for BUF-IN rdma reg\n", 666 iser_err("%d references remain for BUF-IN rdma reg\n",
@@ -671,8 +668,8 @@ void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task *iser_ctask)
671 } 668 }
672 } 669 }
673 670
674 if (iser_ctask->dir[ISER_DIR_OUT]) { 671 if (iser_task->dir[ISER_DIR_OUT]) {
675 regd = &iser_ctask->rdma_regd[ISER_DIR_OUT]; 672 regd = &iser_task->rdma_regd[ISER_DIR_OUT];
676 deferred = iser_regd_buff_release(regd); 673 deferred = iser_regd_buff_release(regd);
677 if (deferred) { 674 if (deferred) {
678 iser_err("%d references remain for BUF-OUT rdma reg\n", 675 iser_err("%d references remain for BUF-OUT rdma reg\n",
@@ -682,7 +679,7 @@ void iser_ctask_rdma_finalize(struct iscsi_iser_cmd_task *iser_ctask)
682 679
683 /* if the data was unaligned, it was already unmapped and then copied */ 680 /* if the data was unaligned, it was already unmapped and then copied */
684 if (is_rdma_aligned) 681 if (is_rdma_aligned)
685 iser_dma_unmap_task_data(iser_ctask); 682 iser_dma_unmap_task_data(iser_task);
686} 683}
687 684
688void iser_dto_buffs_release(struct iser_dto *dto) 685void iser_dto_buffs_release(struct iser_dto *dto)
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index cac50c4dc159..b9453d068e9d 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -28,8 +28,6 @@
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE. 30 * SOFTWARE.
31 *
32 * $Id: iser_memory.c 6964 2006-05-07 11:11:43Z ogerlitz $
33 */ 31 */
34#include <linux/module.h> 32#include <linux/module.h>
35#include <linux/kernel.h> 33#include <linux/kernel.h>
@@ -101,13 +99,13 @@ void iser_reg_single(struct iser_device *device,
101/** 99/**
102 * iser_start_rdma_unaligned_sg 100 * iser_start_rdma_unaligned_sg
103 */ 101 */
104static int iser_start_rdma_unaligned_sg(struct iscsi_iser_cmd_task *iser_ctask, 102static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
105 enum iser_data_dir cmd_dir) 103 enum iser_data_dir cmd_dir)
106{ 104{
107 int dma_nents; 105 int dma_nents;
108 struct ib_device *dev; 106 struct ib_device *dev;
109 char *mem = NULL; 107 char *mem = NULL;
110 struct iser_data_buf *data = &iser_ctask->data[cmd_dir]; 108 struct iser_data_buf *data = &iser_task->data[cmd_dir];
111 unsigned long cmd_data_len = data->data_len; 109 unsigned long cmd_data_len = data->data_len;
112 110
113 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 111 if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
@@ -140,37 +138,37 @@ static int iser_start_rdma_unaligned_sg(struct iscsi_iser_cmd_task *iser_ctask,
140 } 138 }
141 } 139 }
142 140
143 sg_init_one(&iser_ctask->data_copy[cmd_dir].sg_single, mem, cmd_data_len); 141 sg_init_one(&iser_task->data_copy[cmd_dir].sg_single, mem, cmd_data_len);
144 iser_ctask->data_copy[cmd_dir].buf = 142 iser_task->data_copy[cmd_dir].buf =
145 &iser_ctask->data_copy[cmd_dir].sg_single; 143 &iser_task->data_copy[cmd_dir].sg_single;
146 iser_ctask->data_copy[cmd_dir].size = 1; 144 iser_task->data_copy[cmd_dir].size = 1;
147 145
148 iser_ctask->data_copy[cmd_dir].copy_buf = mem; 146 iser_task->data_copy[cmd_dir].copy_buf = mem;
149 147
150 dev = iser_ctask->iser_conn->ib_conn->device->ib_device; 148 dev = iser_task->iser_conn->ib_conn->device->ib_device;
151 dma_nents = ib_dma_map_sg(dev, 149 dma_nents = ib_dma_map_sg(dev,
152 &iser_ctask->data_copy[cmd_dir].sg_single, 150 &iser_task->data_copy[cmd_dir].sg_single,
153 1, 151 1,
154 (cmd_dir == ISER_DIR_OUT) ? 152 (cmd_dir == ISER_DIR_OUT) ?
155 DMA_TO_DEVICE : DMA_FROM_DEVICE); 153 DMA_TO_DEVICE : DMA_FROM_DEVICE);
156 BUG_ON(dma_nents == 0); 154 BUG_ON(dma_nents == 0);
157 155
158 iser_ctask->data_copy[cmd_dir].dma_nents = dma_nents; 156 iser_task->data_copy[cmd_dir].dma_nents = dma_nents;
159 return 0; 157 return 0;
160} 158}
161 159
162/** 160/**
163 * iser_finalize_rdma_unaligned_sg 161 * iser_finalize_rdma_unaligned_sg
164 */ 162 */
165void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_cmd_task *iser_ctask, 163void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
166 enum iser_data_dir cmd_dir) 164 enum iser_data_dir cmd_dir)
167{ 165{
168 struct ib_device *dev; 166 struct ib_device *dev;
169 struct iser_data_buf *mem_copy; 167 struct iser_data_buf *mem_copy;
170 unsigned long cmd_data_len; 168 unsigned long cmd_data_len;
171 169
172 dev = iser_ctask->iser_conn->ib_conn->device->ib_device; 170 dev = iser_task->iser_conn->ib_conn->device->ib_device;
173 mem_copy = &iser_ctask->data_copy[cmd_dir]; 171 mem_copy = &iser_task->data_copy[cmd_dir];
174 172
175 ib_dma_unmap_sg(dev, &mem_copy->sg_single, 1, 173 ib_dma_unmap_sg(dev, &mem_copy->sg_single, 1,
176 (cmd_dir == ISER_DIR_OUT) ? 174 (cmd_dir == ISER_DIR_OUT) ?
@@ -186,8 +184,8 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_cmd_task *iser_ctask,
186 /* copy back read RDMA to unaligned sg */ 184 /* copy back read RDMA to unaligned sg */
187 mem = mem_copy->copy_buf; 185 mem = mem_copy->copy_buf;
188 186
189 sgl = (struct scatterlist *)iser_ctask->data[ISER_DIR_IN].buf; 187 sgl = (struct scatterlist *)iser_task->data[ISER_DIR_IN].buf;
190 sg_size = iser_ctask->data[ISER_DIR_IN].size; 188 sg_size = iser_task->data[ISER_DIR_IN].size;
191 189
192 p = mem; 190 p = mem;
193 for_each_sg(sgl, sg, sg_size, i) { 191 for_each_sg(sgl, sg, sg_size, i) {
@@ -200,7 +198,7 @@ void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_cmd_task *iser_ctask,
200 } 198 }
201 } 199 }
202 200
203 cmd_data_len = iser_ctask->data[cmd_dir].data_len; 201 cmd_data_len = iser_task->data[cmd_dir].data_len;
204 202
205 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 203 if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
206 free_pages((unsigned long)mem_copy->copy_buf, 204 free_pages((unsigned long)mem_copy->copy_buf,
@@ -378,15 +376,15 @@ static void iser_page_vec_build(struct iser_data_buf *data,
378 } 376 }
379} 377}
380 378
381int iser_dma_map_task_data(struct iscsi_iser_cmd_task *iser_ctask, 379int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
382 struct iser_data_buf *data, 380 struct iser_data_buf *data,
383 enum iser_data_dir iser_dir, 381 enum iser_data_dir iser_dir,
384 enum dma_data_direction dma_dir) 382 enum dma_data_direction dma_dir)
385{ 383{
386 struct ib_device *dev; 384 struct ib_device *dev;
387 385
388 iser_ctask->dir[iser_dir] = 1; 386 iser_task->dir[iser_dir] = 1;
389 dev = iser_ctask->iser_conn->ib_conn->device->ib_device; 387 dev = iser_task->iser_conn->ib_conn->device->ib_device;
390 388
391 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); 389 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir);
392 if (data->dma_nents == 0) { 390 if (data->dma_nents == 0) {
@@ -396,20 +394,20 @@ int iser_dma_map_task_data(struct iscsi_iser_cmd_task *iser_ctask,
396 return 0; 394 return 0;
397} 395}
398 396
399void iser_dma_unmap_task_data(struct iscsi_iser_cmd_task *iser_ctask) 397void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task)
400{ 398{
401 struct ib_device *dev; 399 struct ib_device *dev;
402 struct iser_data_buf *data; 400 struct iser_data_buf *data;
403 401
404 dev = iser_ctask->iser_conn->ib_conn->device->ib_device; 402 dev = iser_task->iser_conn->ib_conn->device->ib_device;
405 403
406 if (iser_ctask->dir[ISER_DIR_IN]) { 404 if (iser_task->dir[ISER_DIR_IN]) {
407 data = &iser_ctask->data[ISER_DIR_IN]; 405 data = &iser_task->data[ISER_DIR_IN];
408 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE); 406 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE);
409 } 407 }
410 408
411 if (iser_ctask->dir[ISER_DIR_OUT]) { 409 if (iser_task->dir[ISER_DIR_OUT]) {
412 data = &iser_ctask->data[ISER_DIR_OUT]; 410 data = &iser_task->data[ISER_DIR_OUT];
413 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_TO_DEVICE); 411 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_TO_DEVICE);
414 } 412 }
415} 413}
@@ -420,21 +418,21 @@ void iser_dma_unmap_task_data(struct iscsi_iser_cmd_task *iser_ctask)
420 * 418 *
421 * returns 0 on success, errno code on failure 419 * returns 0 on success, errno code on failure
422 */ 420 */
423int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *iser_ctask, 421int iser_reg_rdma_mem(struct iscsi_iser_task *iser_task,
424 enum iser_data_dir cmd_dir) 422 enum iser_data_dir cmd_dir)
425{ 423{
426 struct iscsi_conn *iscsi_conn = iser_ctask->iser_conn->iscsi_conn; 424 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn;
427 struct iser_conn *ib_conn = iser_ctask->iser_conn->ib_conn; 425 struct iser_conn *ib_conn = iser_task->iser_conn->ib_conn;
428 struct iser_device *device = ib_conn->device; 426 struct iser_device *device = ib_conn->device;
429 struct ib_device *ibdev = device->ib_device; 427 struct ib_device *ibdev = device->ib_device;
430 struct iser_data_buf *mem = &iser_ctask->data[cmd_dir]; 428 struct iser_data_buf *mem = &iser_task->data[cmd_dir];
431 struct iser_regd_buf *regd_buf; 429 struct iser_regd_buf *regd_buf;
432 int aligned_len; 430 int aligned_len;
433 int err; 431 int err;
434 int i; 432 int i;
435 struct scatterlist *sg; 433 struct scatterlist *sg;
436 434
437 regd_buf = &iser_ctask->rdma_regd[cmd_dir]; 435 regd_buf = &iser_task->rdma_regd[cmd_dir];
438 436
439 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 437 aligned_len = iser_data_buf_aligned_len(mem, ibdev);
440 if (aligned_len != mem->dma_nents) { 438 if (aligned_len != mem->dma_nents) {
@@ -444,13 +442,13 @@ int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *iser_ctask,
444 iser_data_buf_dump(mem, ibdev); 442 iser_data_buf_dump(mem, ibdev);
445 443
446 /* unmap the command data before accessing it */ 444 /* unmap the command data before accessing it */
447 iser_dma_unmap_task_data(iser_ctask); 445 iser_dma_unmap_task_data(iser_task);
448 446
449 /* allocate copy buf, if we are writing, copy the */ 447 /* allocate copy buf, if we are writing, copy the */
450 /* unaligned scatterlist, dma map the copy */ 448 /* unaligned scatterlist, dma map the copy */
451 if (iser_start_rdma_unaligned_sg(iser_ctask, cmd_dir) != 0) 449 if (iser_start_rdma_unaligned_sg(iser_task, cmd_dir) != 0)
452 return -ENOMEM; 450 return -ENOMEM;
453 mem = &iser_ctask->data_copy[cmd_dir]; 451 mem = &iser_task->data_copy[cmd_dir];
454 } 452 }
455 453
456 /* if there a single dma entry, FMR is not needed */ 454 /* if there a single dma entry, FMR is not needed */
@@ -474,8 +472,9 @@ int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *iser_ctask,
474 err = iser_reg_page_vec(ib_conn, ib_conn->page_vec, &regd_buf->reg); 472 err = iser_reg_page_vec(ib_conn, ib_conn->page_vec, &regd_buf->reg);
475 if (err) { 473 if (err) {
476 iser_data_buf_dump(mem, ibdev); 474 iser_data_buf_dump(mem, ibdev);
477 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", mem->dma_nents, 475 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n",
478 ntoh24(iser_ctask->desc.iscsi_header.dlength)); 476 mem->dma_nents,
477 ntoh24(iser_task->desc.iscsi_header.dlength));
479 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", 478 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
480 ib_conn->page_vec->data_size, ib_conn->page_vec->length, 479 ib_conn->page_vec->data_size, ib_conn->page_vec->length,
481 ib_conn->page_vec->offset); 480 ib_conn->page_vec->offset);
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index d19cfe605ebb..3a917c1f796f 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -29,8 +29,6 @@
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE. 31 * SOFTWARE.
32 *
33 * $Id: iser_verbs.c 7051 2006-05-10 12:29:11Z ogerlitz $
34 */ 32 */
35#include <linux/kernel.h> 33#include <linux/kernel.h>
36#include <linux/module.h> 34#include <linux/module.h>
@@ -325,7 +323,18 @@ static void iser_conn_release(struct iser_conn *ib_conn)
325 iser_device_try_release(device); 323 iser_device_try_release(device);
326 if (ib_conn->iser_conn) 324 if (ib_conn->iser_conn)
327 ib_conn->iser_conn->ib_conn = NULL; 325 ib_conn->iser_conn->ib_conn = NULL;
328 kfree(ib_conn); 326 iscsi_destroy_endpoint(ib_conn->ep);
327}
328
329void iser_conn_get(struct iser_conn *ib_conn)
330{
331 atomic_inc(&ib_conn->refcount);
332}
333
334void iser_conn_put(struct iser_conn *ib_conn)
335{
336 if (atomic_dec_and_test(&ib_conn->refcount))
337 iser_conn_release(ib_conn);
329} 338}
330 339
331/** 340/**
@@ -349,7 +358,7 @@ void iser_conn_terminate(struct iser_conn *ib_conn)
349 wait_event_interruptible(ib_conn->wait, 358 wait_event_interruptible(ib_conn->wait,
350 ib_conn->state == ISER_CONN_DOWN); 359 ib_conn->state == ISER_CONN_DOWN);
351 360
352 iser_conn_release(ib_conn); 361 iser_conn_put(ib_conn);
353} 362}
354 363
355static void iser_connect_error(struct rdma_cm_id *cma_id) 364static void iser_connect_error(struct rdma_cm_id *cma_id)
@@ -483,24 +492,15 @@ static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *eve
483 return ret; 492 return ret;
484} 493}
485 494
486int iser_conn_init(struct iser_conn **ibconn) 495void iser_conn_init(struct iser_conn *ib_conn)
487{ 496{
488 struct iser_conn *ib_conn;
489
490 ib_conn = kzalloc(sizeof *ib_conn, GFP_KERNEL);
491 if (!ib_conn) {
492 iser_err("can't alloc memory for struct iser_conn\n");
493 return -ENOMEM;
494 }
495 ib_conn->state = ISER_CONN_INIT; 497 ib_conn->state = ISER_CONN_INIT;
496 init_waitqueue_head(&ib_conn->wait); 498 init_waitqueue_head(&ib_conn->wait);
497 atomic_set(&ib_conn->post_recv_buf_count, 0); 499 atomic_set(&ib_conn->post_recv_buf_count, 0);
498 atomic_set(&ib_conn->post_send_buf_count, 0); 500 atomic_set(&ib_conn->post_send_buf_count, 0);
501 atomic_set(&ib_conn->refcount, 1);
499 INIT_LIST_HEAD(&ib_conn->conn_list); 502 INIT_LIST_HEAD(&ib_conn->conn_list);
500 spin_lock_init(&ib_conn->lock); 503 spin_lock_init(&ib_conn->lock);
501
502 *ibconn = ib_conn;
503 return 0;
504} 504}
505 505
506 /** 506 /**
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index 435145709dd6..ed7c5f72cb8b 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -28,8 +28,6 @@
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE. 30 * SOFTWARE.
31 *
32 * $Id: ib_srp.c 3932 2005-11-01 17:19:29Z roland $
33 */ 31 */
34 32
35#include <linux/module.h> 33#include <linux/module.h>
@@ -49,8 +47,6 @@
49#include <scsi/srp.h> 47#include <scsi/srp.h>
50#include <scsi/scsi_transport_srp.h> 48#include <scsi/scsi_transport_srp.h>
51 49
52#include <rdma/ib_cache.h>
53
54#include "ib_srp.h" 50#include "ib_srp.h"
55 51
56#define DRV_NAME "ib_srp" 52#define DRV_NAME "ib_srp"
@@ -183,10 +179,10 @@ static int srp_init_qp(struct srp_target_port *target,
183 if (!attr) 179 if (!attr)
184 return -ENOMEM; 180 return -ENOMEM;
185 181
186 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev, 182 ret = ib_find_pkey(target->srp_host->srp_dev->dev,
187 target->srp_host->port, 183 target->srp_host->port,
188 be16_to_cpu(target->path.pkey), 184 be16_to_cpu(target->path.pkey),
189 &attr->pkey_index); 185 &attr->pkey_index);
190 if (ret) 186 if (ret)
191 goto out; 187 goto out;
192 188
@@ -1883,8 +1879,7 @@ static ssize_t srp_create_target(struct device *dev,
1883 if (ret) 1879 if (ret)
1884 goto err; 1880 goto err;
1885 1881
1886 ib_get_cached_gid(host->srp_dev->dev, host->port, 0, 1882 ib_query_gid(host->srp_dev->dev, host->port, 0, &target->path.sgid);
1887 &target->path.sgid);
1888 1883
1889 shost_printk(KERN_DEBUG, target->scsi_host, PFX 1884 shost_printk(KERN_DEBUG, target->scsi_host, PFX
1890 "new target: id_ext %016llx ioc_guid %016llx pkey %04x " 1885 "new target: id_ext %016llx ioc_guid %016llx pkey %04x "
diff --git a/drivers/infiniband/ulp/srp/ib_srp.h b/drivers/infiniband/ulp/srp/ib_srp.h
index 63d2ae724061..e185b907fc12 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.h
+++ b/drivers/infiniband/ulp/srp/ib_srp.h
@@ -28,8 +28,6 @@
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE. 30 * SOFTWARE.
31 *
32 * $Id: ib_srp.h 3932 2005-11-01 17:19:29Z roland $
33 */ 31 */
34 32
35#ifndef IB_SRP_H 33#ifndef IB_SRP_H
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-08 07:26:20 -0500