aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDave Olson <dave.olson@qlogic.com>2008-04-17 00:01:12 -0400
committerRoland Dreier <rolandd@cisco.com>2008-04-17 00:01:12 -0400
commit826d801009fb3c82832f2d92149446cce354bf61 (patch)
tree564534dee75be00b019b78b130ec6eb7e5a81e7d
parent5d1ce03dd335abaef50dc615137cac2a22c5cee0 (diff)
IB/ipath: Enable 4KB MTU
Enable use of 4KB MTU. Since the driver uses more pinned memory for receive buffers when the 4KB MTU is enabled, whether or not the fabric supports that MTU, add a "mtu4096" module parameter that can be used to limit the MTU to 2KB when it is known that 4KB MTUs can't be used anyway. Signed-off-by: Dave Olson <dave.olson@qlogic.com> Signed-off-by: Roland Dreier <rolandd@cisco.com>
-rw-r--r--drivers/infiniband/hw/ipath/ipath_driver.c31
-rw-r--r--drivers/infiniband/hw/ipath/ipath_file_ops.c7
-rw-r--r--drivers/infiniband/hw/ipath/ipath_iba6120.c10
-rw-r--r--drivers/infiniband/hw/ipath/ipath_init_chip.c30
-rw-r--r--drivers/infiniband/hw/ipath/ipath_kernel.h1
-rw-r--r--drivers/infiniband/hw/ipath/ipath_mad.c12
-rw-r--r--drivers/infiniband/hw/ipath/ipath_qp.c12
-rw-r--r--drivers/infiniband/hw/ipath/ipath_verbs.c7
8 files changed, 45 insertions, 65 deletions
diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c
index 367f2a38744e..7121fe84ff8b 100644
--- a/drivers/infiniband/hw/ipath/ipath_driver.c
+++ b/drivers/infiniband/hw/ipath/ipath_driver.c
@@ -73,6 +73,10 @@ module_param_named(debug, ipath_debug, uint, S_IWUSR | S_IRUGO);
73MODULE_PARM_DESC(debug, "mask for debug prints"); 73MODULE_PARM_DESC(debug, "mask for debug prints");
74EXPORT_SYMBOL_GPL(ipath_debug); 74EXPORT_SYMBOL_GPL(ipath_debug);
75 75
76unsigned ipath_mtu4096 = 1; /* max 4KB IB mtu by default, if supported */
77module_param_named(mtu4096, ipath_mtu4096, uint, S_IRUGO);
78MODULE_PARM_DESC(mtu4096, "enable MTU of 4096 bytes, if supported");
79
76MODULE_LICENSE("GPL"); 80MODULE_LICENSE("GPL");
77MODULE_AUTHOR("QLogic <support@pathscale.com>"); 81MODULE_AUTHOR("QLogic <support@pathscale.com>");
78MODULE_DESCRIPTION("QLogic InfiniPath driver"); 82MODULE_DESCRIPTION("QLogic InfiniPath driver");
@@ -1800,7 +1804,7 @@ int ipath_set_mtu(struct ipath_devdata *dd, u16 arg)
1800 * piosize). We check that it's one of the valid IB sizes. 1804 * piosize). We check that it's one of the valid IB sizes.
1801 */ 1805 */
1802 if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 && 1806 if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 &&
1803 arg != 4096) { 1807 (arg != 4096 || !ipath_mtu4096)) {
1804 ipath_dbg("Trying to set invalid mtu %u, failing\n", arg); 1808 ipath_dbg("Trying to set invalid mtu %u, failing\n", arg);
1805 ret = -EINVAL; 1809 ret = -EINVAL;
1806 goto bail; 1810 goto bail;
@@ -1816,6 +1820,8 @@ int ipath_set_mtu(struct ipath_devdata *dd, u16 arg)
1816 if (arg >= (piosize - IPATH_PIO_MAXIBHDR)) { 1820 if (arg >= (piosize - IPATH_PIO_MAXIBHDR)) {
1817 /* Only if it's not the initial value (or reset to it) */ 1821 /* Only if it's not the initial value (or reset to it) */
1818 if (piosize != dd->ipath_init_ibmaxlen) { 1822 if (piosize != dd->ipath_init_ibmaxlen) {
1823 if (arg > piosize && arg <= dd->ipath_init_ibmaxlen)
1824 piosize = dd->ipath_init_ibmaxlen;
1819 dd->ipath_ibmaxlen = piosize; 1825 dd->ipath_ibmaxlen = piosize;
1820 changed = 1; 1826 changed = 1;
1821 } 1827 }
@@ -1829,24 +1835,17 @@ int ipath_set_mtu(struct ipath_devdata *dd, u16 arg)
1829 } 1835 }
1830 1836
1831 if (changed) { 1837 if (changed) {
1838 u64 ibc = dd->ipath_ibcctrl, ibdw;
1832 /* 1839 /*
1833 * set the IBC maxpktlength to the size of our pio 1840 * update our housekeeping variables, and set IBC max
1834 * buffers in words 1841 * size, same as init code; max IBC is max we allow in
1842 * buffer, less the qword pbc, plus 1 for ICRC, in dwords
1835 */ 1843 */
1836 u64 ibc = dd->ipath_ibcctrl; 1844 dd->ipath_ibmaxlen = piosize - 2 * sizeof(u32);
1845 ibdw = (dd->ipath_ibmaxlen >> 2) + 1;
1837 ibc &= ~(INFINIPATH_IBCC_MAXPKTLEN_MASK << 1846 ibc &= ~(INFINIPATH_IBCC_MAXPKTLEN_MASK <<
1838 INFINIPATH_IBCC_MAXPKTLEN_SHIFT); 1847 dd->ibcc_mpl_shift);
1839 1848 ibc |= ibdw << dd->ibcc_mpl_shift;
1840 piosize = piosize - 2 * sizeof(u32); /* ignore pbc */
1841 dd->ipath_ibmaxlen = piosize;
1842 piosize /= sizeof(u32); /* in words */
1843 /*
1844 * for ICRC, which we only send in diag test pkt mode, and
1845 * we don't need to worry about that for mtu
1846 */
1847 piosize += 1;
1848
1849 ibc |= piosize << INFINIPATH_IBCC_MAXPKTLEN_SHIFT;
1850 dd->ipath_ibcctrl = ibc; 1849 dd->ipath_ibcctrl = ibc;
1851 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, 1850 ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
1852 dd->ipath_ibcctrl); 1851 dd->ipath_ibcctrl);
diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c
index 219b62d32eed..cddf29b9554e 100644
--- a/drivers/infiniband/hw/ipath/ipath_file_ops.c
+++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c
@@ -219,7 +219,12 @@ static int ipath_get_base_info(struct file *fp,
219 kinfo->spi_pioalign = dd->ipath_palign; 219 kinfo->spi_pioalign = dd->ipath_palign;
220 220
221 kinfo->spi_qpair = IPATH_KD_QP; 221 kinfo->spi_qpair = IPATH_KD_QP;
222 kinfo->spi_piosize = dd->ipath_ibmaxlen; 222 /*
223 * user mode PIO buffers are always 2KB, even when 4KB can
224 * be received, and sent via the kernel; this is ibmaxlen
225 * for 2K MTU.
226 */
227 kinfo->spi_piosize = dd->ipath_piosize2k - 2 * sizeof(u32);
223 kinfo->spi_mtu = dd->ipath_ibmaxlen; /* maxlen, not ibmtu */ 228 kinfo->spi_mtu = dd->ipath_ibmaxlen; /* maxlen, not ibmtu */
224 kinfo->spi_port = pd->port_port; 229 kinfo->spi_port = pd->port_port;
225 kinfo->spi_subport = subport_fp(fp); 230 kinfo->spi_subport = subport_fp(fp);
diff --git a/drivers/infiniband/hw/ipath/ipath_iba6120.c b/drivers/infiniband/hw/ipath/ipath_iba6120.c
index 828066e20ad7..a9fc80409e57 100644
--- a/drivers/infiniband/hw/ipath/ipath_iba6120.c
+++ b/drivers/infiniband/hw/ipath/ipath_iba6120.c
@@ -1441,17 +1441,13 @@ static int ipath_pe_early_init(struct ipath_devdata *dd)
1441 dd->ipath_egrtidbase = (u64 __iomem *) 1441 dd->ipath_egrtidbase = (u64 __iomem *)
1442 ((char __iomem *) dd->ipath_kregbase + dd->ipath_rcvegrbase); 1442 ((char __iomem *) dd->ipath_kregbase + dd->ipath_rcvegrbase);
1443 1443
1444 /* 1444 dd->ipath_rcvegrbufsize = ipath_mtu4096 ? 4096 : 2048;
1445 * To truly support a 4KB MTU (for usermode), we need to
1446 * bump this to a larger value. For now, we use them for
1447 * the kernel only.
1448 */
1449 dd->ipath_rcvegrbufsize = 2048;
1450 /* 1445 /*
1451 * the min() check here is currently a nop, but it may not always 1446 * the min() check here is currently a nop, but it may not always
1452 * be, depending on just how we do ipath_rcvegrbufsize 1447 * be, depending on just how we do ipath_rcvegrbufsize
1453 */ 1448 */
1454 dd->ipath_ibmaxlen = min(dd->ipath_piosize2k, 1449 dd->ipath_ibmaxlen = min(ipath_mtu4096 ? dd->ipath_piosize4k :
1450 dd->ipath_piosize2k,
1455 dd->ipath_rcvegrbufsize + 1451 dd->ipath_rcvegrbufsize +
1456 (dd->ipath_rcvhdrentsize << 2)); 1452 (dd->ipath_rcvhdrentsize << 2));
1457 dd->ipath_init_ibmaxlen = dd->ipath_ibmaxlen; 1453 dd->ipath_init_ibmaxlen = dd->ipath_ibmaxlen;
diff --git a/drivers/infiniband/hw/ipath/ipath_init_chip.c b/drivers/infiniband/hw/ipath/ipath_init_chip.c
index 5428affa9102..f0d7848d9bb1 100644
--- a/drivers/infiniband/hw/ipath/ipath_init_chip.c
+++ b/drivers/infiniband/hw/ipath/ipath_init_chip.c
@@ -155,24 +155,13 @@ static int bringup_link(struct ipath_devdata *dd)
155 dd->ipath_control); 155 dd->ipath_control);
156 156
157 /* 157 /*
158 * Note that prior to try 14 or 15 of IB, the credit scaling 158 * set initial max size pkt IBC will send, including ICRC; it's the
159 * wasn't working, because it was swapped for writes with the 159 * PIO buffer size in dwords, less 1; also see ipath_set_mtu()
160 * 1 bit default linkstate field
161 */ 160 */
161 val = (dd->ipath_ibmaxlen >> 2) + 1;
162 ibc = val << dd->ibcc_mpl_shift;
162 163
163 /* ignore pbc and align word */ 164 /* flowcontrolwatermark is in units of KBytes */
164 val = dd->ipath_piosize2k - 2 * sizeof(u32);
165 /*
166 * for ICRC, which we only send in diag test pkt mode, and we
167 * don't need to worry about that for mtu
168 */
169 val += 1;
170 /*
171 * Set the IBC maxpktlength to the size of our pio buffers the
172 * maxpktlength is in words. This is *not* the IB data MTU.
173 */
174 ibc = (val / sizeof(u32)) << INFINIPATH_IBCC_MAXPKTLEN_SHIFT;
175 /* in KB */
176 ibc |= 0x5ULL << INFINIPATH_IBCC_FLOWCTRLWATERMARK_SHIFT; 165 ibc |= 0x5ULL << INFINIPATH_IBCC_FLOWCTRLWATERMARK_SHIFT;
177 /* 166 /*
178 * How often flowctrl sent. More or less in usecs; balance against 167 * How often flowctrl sent. More or less in usecs; balance against
@@ -295,12 +284,9 @@ static int init_chip_first(struct ipath_devdata *dd,
295 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpiosize); 284 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpiosize);
296 dd->ipath_piosize2k = val & ~0U; 285 dd->ipath_piosize2k = val & ~0U;
297 dd->ipath_piosize4k = val >> 32; 286 dd->ipath_piosize4k = val >> 32;
298 /* 287 if (dd->ipath_piosize4k == 0 && ipath_mtu4096)
299 * Note: the chips support a maximum MTU of 4096, but the driver 288 ipath_mtu4096 = 0; /* 4KB not supported by this chip */
300 * hasn't implemented this feature yet, so set the initial value 289 dd->ipath_ibmtu = ipath_mtu4096 ? 4096 : 2048;
301 * to 2048.
302 */
303 dd->ipath_ibmtu = 2048;
304 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpiobufcnt); 290 val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_sendpiobufcnt);
305 dd->ipath_piobcnt2k = val & ~0U; 291 dd->ipath_piobcnt2k = val & ~0U;
306 dd->ipath_piobcnt4k = val >> 32; 292 dd->ipath_piobcnt4k = val >> 32;
diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h
index 59dc89516243..70c0a0dd6939 100644
--- a/drivers/infiniband/hw/ipath/ipath_kernel.h
+++ b/drivers/infiniband/hw/ipath/ipath_kernel.h
@@ -1066,6 +1066,7 @@ dma_addr_t ipath_map_single(struct pci_dev *, void *, size_t, int);
1066#endif 1066#endif
1067 1067
1068extern unsigned ipath_debug; /* debugging bit mask */ 1068extern unsigned ipath_debug; /* debugging bit mask */
1069extern unsigned ipath_mtu4096;
1069 1070
1070#define IPATH_MAX_PARITY_ATTEMPTS 10000 /* max times to try recovery */ 1071#define IPATH_MAX_PARITY_ATTEMPTS 10000 /* max times to try recovery */
1071 1072
diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c
index b34b91d3723a..aca876bae1c4 100644
--- a/drivers/infiniband/hw/ipath/ipath_mad.c
+++ b/drivers/infiniband/hw/ipath/ipath_mad.c
@@ -292,13 +292,9 @@ static int recv_subn_get_portinfo(struct ib_smp *smp,
292 /* pip->vl_arb_high_cap; // only one VL */ 292 /* pip->vl_arb_high_cap; // only one VL */
293 /* pip->vl_arb_low_cap; // only one VL */ 293 /* pip->vl_arb_low_cap; // only one VL */
294 /* InitTypeReply = 0 */ 294 /* InitTypeReply = 0 */
295 /* 295 /* our mtu cap depends on whether 4K MTU enabled or not */
296 * Note: the chips support a maximum MTU of 4096, but the driver 296 pip->inittypereply_mtucap = ipath_mtu4096 ? IB_MTU_4096 : IB_MTU_2048;
297 * hasn't implemented this feature yet, so set the maximum value 297 /* HCAs ignore VLStallCount and HOQLife */
298 * to 2048.
299 */
300 pip->inittypereply_mtucap = IB_MTU_2048;
301 // HCAs ignore VLStallCount and HOQLife
302 /* pip->vlstallcnt_hoqlife; */ 298 /* pip->vlstallcnt_hoqlife; */
303 pip->operationalvl_pei_peo_fpi_fpo = 0x10; /* OVLs = 1 */ 299 pip->operationalvl_pei_peo_fpi_fpo = 0x10; /* OVLs = 1 */
304 pip->mkey_violations = cpu_to_be16(dev->mkey_violations); 300 pip->mkey_violations = cpu_to_be16(dev->mkey_violations);
@@ -491,6 +487,8 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
491 mtu = 2048; 487 mtu = 2048;
492 break; 488 break;
493 case IB_MTU_4096: 489 case IB_MTU_4096:
490 if (!ipath_mtu4096)
491 goto err;
494 mtu = 4096; 492 mtu = 4096;
495 break; 493 break;
496 default: 494 default:
diff --git a/drivers/infiniband/hw/ipath/ipath_qp.c b/drivers/infiniband/hw/ipath/ipath_qp.c
index 087ed3166479..5c8094a03643 100644
--- a/drivers/infiniband/hw/ipath/ipath_qp.c
+++ b/drivers/infiniband/hw/ipath/ipath_qp.c
@@ -516,13 +516,13 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
516 goto inval; 516 goto inval;
517 517
518 /* 518 /*
519 * Note: the chips support a maximum MTU of 4096, but the driver 519 * don't allow invalid Path MTU values or greater than 2048
520 * hasn't implemented this feature yet, so don't allow Path MTU 520 * unless we are configured for a 4KB MTU
521 * values greater than 2048.
522 */ 521 */
523 if (attr_mask & IB_QP_PATH_MTU) 522 if ((attr_mask & IB_QP_PATH_MTU) &&
524 if (attr->path_mtu > IB_MTU_2048) 523 (ib_mtu_enum_to_int(attr->path_mtu) == -1 ||
525 goto inval; 524 (attr->path_mtu > IB_MTU_2048 && !ipath_mtu4096)))
525 goto inval;
526 526
527 if (attr_mask & IB_QP_PATH_MIG_STATE) 527 if (attr_mask & IB_QP_PATH_MIG_STATE)
528 if (attr->path_mig_state != IB_MIG_MIGRATED && 528 if (attr->path_mig_state != IB_MIG_MIGRATED &&
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.c b/drivers/infiniband/hw/ipath/ipath_verbs.c
index 32d8f882e56c..012ccb4f9a37 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.c
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.c
@@ -1201,12 +1201,7 @@ static int ipath_query_port(struct ib_device *ibdev,
1201 props->max_vl_num = 1; /* VLCap = VL0 */ 1201 props->max_vl_num = 1; /* VLCap = VL0 */
1202 props->init_type_reply = 0; 1202 props->init_type_reply = 0;
1203 1203
1204 /* 1204 props->max_mtu = ipath_mtu4096 ? IB_MTU_4096 : IB_MTU_2048;
1205 * Note: the chip supports a maximum MTU of 4096, but the driver
1206 * hasn't implemented this feature yet, so set the maximum value
1207 * to 2048.
1208 */
1209 props->max_mtu = IB_MTU_2048;
1210 switch (dd->ipath_ibmtu) { 1205 switch (dd->ipath_ibmtu) {
1211 case 4096: 1206 case 4096:
1212 mtu = IB_MTU_4096; 1207 mtu = IB_MTU_4096;