aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNicholas Bellinger <nab@linux-iscsi.org>2013-11-05 16:16:12 -0500
committerNicholas Bellinger <nab@linux-iscsi.org>2013-11-06 15:48:22 -0500
commit95b60f078899a9680688bab909d02169e1fd632e (patch)
tree15af0172ede5d13a482e6d9cebca7a2ae8ab47d2
parent2dd1d53fe004cd3b1e93e96508cfe637e6e31ef5 (diff)
ib_isert: Add support for completion interrupt coalescing
This patch adds support for completion interrupt coalescing that allows only every ISERT_COMP_BATCH_COUNT (8) to set IB_SEND_SIGNALED, thus avoiding completion interrupts for every posted iser_tx_desc. The batch processing is done using a per isert_conn llist that once IB_SEND_SIGNALED has been set is saved to tx_desc->comp_llnode_batch, and completion processing of previously posted iser_tx_descs is done in a single shot from within isert_send_completion() code. Note this is only done for response PDUs from ISCSI_OP_SCSI_CMD, and all other control type of PDU responses will force an implicit batch drain to occur. Cc: Or Gerlitz <ogerlitz@mellanox.com> Cc: Sagi Grimberg <sagig@mellanox.com> Cc: Kent Overstreet <kmo@daterainc.com> Cc: Roland Dreier <roland@purestorage.com> Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
-rw-r--r--drivers/infiniband/ulp/isert/ib_isert.c63
-rw-r--r--drivers/infiniband/ulp/isert/ib_isert.h6
2 files changed, 57 insertions, 12 deletions
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index 3591855cc5b5..27708c3430d5 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -22,6 +22,7 @@
22#include <linux/socket.h> 22#include <linux/socket.h>
23#include <linux/in.h> 23#include <linux/in.h>
24#include <linux/in6.h> 24#include <linux/in6.h>
25#include <linux/llist.h>
25#include <rdma/ib_verbs.h> 26#include <rdma/ib_verbs.h>
26#include <rdma/rdma_cm.h> 27#include <rdma/rdma_cm.h>
27#include <target/target_core_base.h> 28#include <target/target_core_base.h>
@@ -489,6 +490,7 @@ isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
489 kref_init(&isert_conn->conn_kref); 490 kref_init(&isert_conn->conn_kref);
490 kref_get(&isert_conn->conn_kref); 491 kref_get(&isert_conn->conn_kref);
491 mutex_init(&isert_conn->conn_mutex); 492 mutex_init(&isert_conn->conn_mutex);
493 mutex_init(&isert_conn->conn_comp_mutex);
492 spin_lock_init(&isert_conn->conn_lock); 494 spin_lock_init(&isert_conn->conn_lock);
493 495
494 cma_id->context = isert_conn; 496 cma_id->context = isert_conn;
@@ -843,14 +845,32 @@ isert_init_tx_hdrs(struct isert_conn *isert_conn,
843} 845}
844 846
845static void 847static void
846isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr) 848isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
849 struct ib_send_wr *send_wr, bool coalesce)
847{ 850{
851 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;
852
848 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 853 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
849 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 854 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc;
850 send_wr->opcode = IB_WR_SEND; 855 send_wr->opcode = IB_WR_SEND;
851 send_wr->send_flags = IB_SEND_SIGNALED; 856 send_wr->sg_list = &tx_desc->tx_sg[0];
852 send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0];
853 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 857 send_wr->num_sge = isert_cmd->tx_desc.num_sge;
858 /*
859 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED
860 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls.
861 */
862 mutex_lock(&isert_conn->conn_comp_mutex);
863 if (coalesce &&
864 ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) {
865 llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist);
866 mutex_unlock(&isert_conn->conn_comp_mutex);
867 return;
868 }
869 isert_conn->conn_comp_batch = 0;
870 tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist);
871 mutex_unlock(&isert_conn->conn_comp_mutex);
872
873 send_wr->send_flags = IB_SEND_SIGNALED;
854} 874}
855 875
856static int 876static int
@@ -1582,8 +1602,8 @@ isert_response_completion(struct iser_tx_desc *tx_desc,
1582} 1602}
1583 1603
1584static void 1604static void
1585isert_send_completion(struct iser_tx_desc *tx_desc, 1605__isert_send_completion(struct iser_tx_desc *tx_desc,
1586 struct isert_conn *isert_conn) 1606 struct isert_conn *isert_conn)
1587{ 1607{
1588 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1608 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
1589 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1609 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
@@ -1624,6 +1644,24 @@ isert_send_completion(struct iser_tx_desc *tx_desc,
1624} 1644}
1625 1645
1626static void 1646static void
1647isert_send_completion(struct iser_tx_desc *tx_desc,
1648 struct isert_conn *isert_conn)
1649{
1650 struct llist_node *llnode = tx_desc->comp_llnode_batch;
1651 struct iser_tx_desc *t;
1652 /*
1653 * Drain coalesced completion llist starting from comp_llnode_batch
1654 * setup in isert_init_send_wr(), and then complete trailing tx_desc.
1655 */
1656 while (llnode) {
1657 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode);
1658 llnode = llist_next(llnode);
1659 __isert_send_completion(t, isert_conn);
1660 }
1661 __isert_send_completion(tx_desc, isert_conn);
1662}
1663
1664static void
1627isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1665isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn)
1628{ 1666{
1629 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1667 struct ib_device *ib_dev = isert_conn->conn_cm_id->device;
@@ -1793,7 +1831,7 @@ isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1793 isert_cmd->tx_desc.num_sge = 2; 1831 isert_cmd->tx_desc.num_sge = 2;
1794 } 1832 }
1795 1833
1796 isert_init_send_wr(isert_cmd, send_wr); 1834 isert_init_send_wr(isert_conn, isert_cmd, send_wr, true);
1797 1835
1798 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1836 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1799 1837
@@ -1813,7 +1851,7 @@ isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
1813 &isert_cmd->tx_desc.iscsi_header, 1851 &isert_cmd->tx_desc.iscsi_header,
1814 nopout_response); 1852 nopout_response);
1815 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1853 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1816 isert_init_send_wr(isert_cmd, send_wr); 1854 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false);
1817 1855
1818 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1856 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1819 1857
@@ -1831,7 +1869,7 @@ isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1831 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1869 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1832 &isert_cmd->tx_desc.iscsi_header); 1870 &isert_cmd->tx_desc.iscsi_header);
1833 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1871 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1834 isert_init_send_wr(isert_cmd, send_wr); 1872 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false);
1835 1873
1836 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1874 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1837 1875
@@ -1849,7 +1887,7 @@ isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1849 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1887 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1850 &isert_cmd->tx_desc.iscsi_header); 1888 &isert_cmd->tx_desc.iscsi_header);
1851 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1889 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1852 isert_init_send_wr(isert_cmd, send_wr); 1890 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false);
1853 1891
1854 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1892 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1855 1893
@@ -1881,7 +1919,7 @@ isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1881 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1919 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1882 isert_cmd->tx_desc.num_sge = 2; 1920 isert_cmd->tx_desc.num_sge = 2;
1883 1921
1884 isert_init_send_wr(isert_cmd, send_wr); 1922 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false);
1885 1923
1886 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1924 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1887 1925
@@ -1921,7 +1959,7 @@ isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1921 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1959 tx_dsg->lkey = isert_conn->conn_mr->lkey;
1922 isert_cmd->tx_desc.num_sge = 2; 1960 isert_cmd->tx_desc.num_sge = 2;
1923 } 1961 }
1924 isert_init_send_wr(isert_cmd, send_wr); 1962 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false);
1925 1963
1926 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1964 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n");
1927 1965
@@ -2309,7 +2347,8 @@ isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2309 iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *) 2347 iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *)
2310 &isert_cmd->tx_desc.iscsi_header); 2348 &isert_cmd->tx_desc.iscsi_header);
2311 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2349 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2312 isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr); 2350 isert_init_send_wr(isert_conn, isert_cmd,
2351 &isert_cmd->tx_desc.send_wr, true);
2313 2352
2314 atomic_inc(&isert_conn->post_send_buf_count); 2353 atomic_inc(&isert_conn->post_send_buf_count);
2315 2354
diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h
index 631f2090f0b8..691f90ff2d83 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.h
+++ b/drivers/infiniband/ulp/isert/ib_isert.h
@@ -43,6 +43,8 @@ struct iser_tx_desc {
43 struct ib_sge tx_sg[2]; 43 struct ib_sge tx_sg[2];
44 int num_sge; 44 int num_sge;
45 struct isert_cmd *isert_cmd; 45 struct isert_cmd *isert_cmd;
46 struct llist_node *comp_llnode_batch;
47 struct llist_node comp_llnode;
46 struct ib_send_wr send_wr; 48 struct ib_send_wr send_wr;
47} __packed; 49} __packed;
48 50
@@ -121,6 +123,10 @@ struct isert_conn {
121 int conn_frwr_pool_size; 123 int conn_frwr_pool_size;
122 /* lock to protect frwr_pool */ 124 /* lock to protect frwr_pool */
123 spinlock_t conn_lock; 125 spinlock_t conn_lock;
126#define ISERT_COMP_BATCH_COUNT 8
127 int conn_comp_batch;
128 struct llist_head conn_comp_llist;
129 struct mutex conn_comp_mutex;
124}; 130};
125 131
126#define ISERT_MAX_CQ 64 132#define ISERT_MAX_CQ 64