aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-02-04 14:13:49 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2018-02-04 14:13:49 -0500
commitd3658c2266012f270da52e3e0365536e394bd3bd (patch)
tree3ea28fa631873cb7bbc9eec4e8405ee886cd779a
parent8ac4840a3c90cf45830b265c0a4d0876358e8f59 (diff)
parent3b28c987fb9547ca9aac73241d0e281cf646387c (diff)
Merge tag 'ntb-4.16' of git://github.com/jonmason/ntb
Pull NTB updates from Jon Mason: "Bug fixes galore, removal of the ntb atom driver, and updates to the ntb tools and tests to support the multi-port interface" * tag 'ntb-4.16' of git://github.com/jonmason/ntb: (37 commits) NTB: ntb_perf: fix cast to restricted __le32 ntb_perf: Fix an error code in perf_copy_chunk() ntb_hw_switchtec: Make function switchtec_ntb_remove() static NTB: ntb_tool: fix memory leak on 'buf' on error exit path NTB: ntb_perf: fix printing of resource_size_t NTB: ntb_hw_idt: Set NTB_TOPO_SWITCH topology NTB: ntb_test: Update ntb_perf tests NTB: ntb_test: Update ntb_tool MW tests NTB: ntb_test: Add ntb_tool Message tests NTB: ntb_test: Update ntb_tool Scratchpad tests NTB: ntb_test: Update ntb_tool DB tests NTB: ntb_test: Update ntb_tool link tests NTB: ntb_test: Add ntb_tool port tests NTB: ntb_test: Safely use paths with whitespace NTB: ntb_perf: Add full multi-port NTB API support NTB: ntb_tool: Add full multi-port NTB API support NTB: ntb_pp: Add full multi-port NTB API support NTB: Fix UB/bug in ntb_mw_get_align() NTB: Set dma mask and dma coherent mask to NTB devices NTB: Rename NTB messaging API methods ...
Diffstat
-rw-r--r--MAINTAINERS2
-rw-r--r--drivers/ntb/hw/amd/ntb_hw_amd.c4
-rw-r--r--drivers/ntb/hw/idt/ntb_hw_idt.c37
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_intel.c313
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_intel.h58
-rw-r--r--drivers/ntb/hw/mscc/ntb_hw_switchtec.c603
-rw-r--r--drivers/ntb/ntb.c4
-rw-r--r--drivers/ntb/ntb_transport.c3
-rw-r--r--drivers/ntb/test/ntb_perf.c1824
-rw-r--r--drivers/ntb/test/ntb_pingpong.c450
-rw-r--r--drivers/ntb/test/ntb_tool.c1825
-rw-r--r--include/linux/ntb.h51
-rw-r--r--include/linux/switchtec.h23
-rwxr-xr-xtools/testing/selftests/ntb/ntb_test.sh307
14 files changed, 3550 insertions, 1954 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index e15f4e201471..b59a8cdfbe66 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -9801,7 +9801,7 @@ F: drivers/ntb/hw/amd/
9801NTB DRIVER CORE 9801NTB DRIVER CORE
9802M: Jon Mason <jdmason@kudzu.us> 9802M: Jon Mason <jdmason@kudzu.us>
9803M: Dave Jiang <dave.jiang@intel.com> 9803M: Dave Jiang <dave.jiang@intel.com>
9804M: Allen Hubbe <Allen.Hubbe@emc.com> 9804M: Allen Hubbe <allenbh@gmail.com>
9805L: linux-ntb@googlegroups.com 9805L: linux-ntb@googlegroups.com
9806S: Supported 9806S: Supported
9807W: https://github.com/jonmason/ntb/wiki 9807W: https://github.com/jonmason/ntb/wiki
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index f0788aae05c9..3cfa46876239 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -1020,6 +1020,10 @@ static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
1020 goto err_dma_mask; 1020 goto err_dma_mask;
1021 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); 1021 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1022 } 1022 }
1023 rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
1024 dma_get_mask(&pdev->dev));
1025 if (rc)
1026 goto err_dma_mask;
1023 1027
1024 ndev->self_mmio = pci_iomap(pdev, 0, 0); 1028 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1025 if (!ndev->self_mmio) { 1029 if (!ndev->self_mmio) {
diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c
index 0cd79f367f7c..8d98872d0983 100644
--- a/drivers/ntb/hw/idt/ntb_hw_idt.c
+++ b/drivers/ntb/hw/idt/ntb_hw_idt.c
@@ -1744,20 +1744,19 @@ static int idt_ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits)
1744 * idt_ntb_msg_read() - read message register with specified index 1744 * idt_ntb_msg_read() - read message register with specified index
1745 * (NTB API callback) 1745 * (NTB API callback)
1746 * @ntb: NTB device context. 1746 * @ntb: NTB device context.
1747 * @midx: Message register index
1748 * @pidx: OUT - Port index of peer device a message retrieved from 1747 * @pidx: OUT - Port index of peer device a message retrieved from
1749 * @msg: OUT - Data 1748 * @midx: Message register index
1750 * 1749 *
1751 * Read data from the specified message register and source register. 1750 * Read data from the specified message register and source register.
1752 * 1751 *
1753 * Return: zero on success, negative error if invalid argument passed. 1752 * Return: inbound message register value.
1754 */ 1753 */
1755static int idt_ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg) 1754static u32 idt_ntb_msg_read(struct ntb_dev *ntb, int *pidx, int midx)
1756{ 1755{
1757 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1756 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1758 1757
1759 if (midx < 0 || IDT_MSG_CNT <= midx) 1758 if (midx < 0 || IDT_MSG_CNT <= midx)
1760 return -EINVAL; 1759 return ~(u32)0;
1761 1760
1762 /* Retrieve source port index of the message */ 1761 /* Retrieve source port index of the message */
1763 if (pidx != NULL) { 1762 if (pidx != NULL) {
@@ -1772,18 +1771,15 @@ static int idt_ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg)
1772 } 1771 }
1773 1772
1774 /* Retrieve data of the corresponding message register */ 1773 /* Retrieve data of the corresponding message register */
1775 if (msg != NULL) 1774 return idt_nt_read(ndev, ntdata_tbl.msgs[midx].in);
1776 *msg = idt_nt_read(ndev, ntdata_tbl.msgs[midx].in);
1777
1778 return 0;
1779} 1775}
1780 1776
1781/* 1777/*
1782 * idt_ntb_msg_write() - write data to the specified message register 1778 * idt_ntb_peer_msg_write() - write data to the specified message register
1783 * (NTB API callback) 1779 * (NTB API callback)
1784 * @ntb: NTB device context. 1780 * @ntb: NTB device context.
1785 * @midx: Message register index
1786 * @pidx: Port index of peer device a message being sent to 1781 * @pidx: Port index of peer device a message being sent to
1782 * @midx: Message register index
1787 * @msg: Data to send 1783 * @msg: Data to send
1788 * 1784 *
1789 * Just try to send data to a peer. Message status register should be 1785 * Just try to send data to a peer. Message status register should be
@@ -1791,7 +1787,8 @@ static int idt_ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg)
1791 * 1787 *
1792 * Return: zero on success, negative error if invalid argument passed. 1788 * Return: zero on success, negative error if invalid argument passed.
1793 */ 1789 */
1794static int idt_ntb_msg_write(struct ntb_dev *ntb, int midx, int pidx, u32 msg) 1790static int idt_ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx,
1791 u32 msg)
1795{ 1792{
1796 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1793 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1797 unsigned long irqflags; 1794 unsigned long irqflags;
@@ -2058,7 +2055,7 @@ static const struct ntb_dev_ops idt_ntb_ops = {
2058 .msg_set_mask = idt_ntb_msg_set_mask, 2055 .msg_set_mask = idt_ntb_msg_set_mask,
2059 .msg_clear_mask = idt_ntb_msg_clear_mask, 2056 .msg_clear_mask = idt_ntb_msg_clear_mask,
2060 .msg_read = idt_ntb_msg_read, 2057 .msg_read = idt_ntb_msg_read,
2061 .msg_write = idt_ntb_msg_write 2058 .peer_msg_write = idt_ntb_peer_msg_write
2062}; 2059};
2063 2060
2064/* 2061/*
@@ -2073,7 +2070,7 @@ static int idt_register_device(struct idt_ntb_dev *ndev)
2073 2070
2074 /* Initialize the rest of NTB device structure and register it */ 2071 /* Initialize the rest of NTB device structure and register it */
2075 ndev->ntb.ops = &idt_ntb_ops; 2072 ndev->ntb.ops = &idt_ntb_ops;
2076 ndev->ntb.topo = NTB_TOPO_PRI; 2073 ndev->ntb.topo = NTB_TOPO_SWITCH;
2077 2074
2078 ret = ntb_register_device(&ndev->ntb); 2075 ret = ntb_register_device(&ndev->ntb);
2079 if (ret != 0) { 2076 if (ret != 0) {
@@ -2269,7 +2266,7 @@ static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf,
2269 "Message data:\n"); 2266 "Message data:\n");
2270 for (idx = 0; idx < IDT_MSG_CNT; idx++) { 2267 for (idx = 0; idx < IDT_MSG_CNT; idx++) {
2271 int src; 2268 int src;
2272 (void)idt_ntb_msg_read(&ndev->ntb, idx, &src, &data); 2269 data = idt_ntb_msg_read(&ndev->ntb, &src, idx);
2273 off += scnprintf(strbuf + off, size - off, 2270 off += scnprintf(strbuf + off, size - off,
2274 "\t%hhu. 0x%08x from peer %hhu (Port %hhu)\n", 2271 "\t%hhu. 0x%08x from peer %hhu (Port %hhu)\n",
2275 idx, data, src, ndev->peers[src].port); 2272 idx, data, src, ndev->peers[src].port);
@@ -2429,7 +2426,7 @@ static int idt_init_pci(struct idt_ntb_dev *ndev)
2429 struct pci_dev *pdev = ndev->ntb.pdev; 2426 struct pci_dev *pdev = ndev->ntb.pdev;
2430 int ret; 2427 int ret;
2431 2428
2432 /* Initialize the bit mask of DMA */ 2429 /* Initialize the bit mask of PCI/NTB DMA */
2433 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 2430 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2434 if (ret != 0) { 2431 if (ret != 0) {
2435 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 2432 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
@@ -2450,6 +2447,12 @@ static int idt_init_pci(struct idt_ntb_dev *ndev)
2450 dev_warn(&pdev->dev, 2447 dev_warn(&pdev->dev,
2451 "Cannot set consistent DMA highmem bit mask\n"); 2448 "Cannot set consistent DMA highmem bit mask\n");
2452 } 2449 }
2450 ret = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
2451 dma_get_mask(&pdev->dev));
2452 if (ret != 0) {
2453 dev_err(&pdev->dev, "Failed to set NTB device DMA bit mask\n");
2454 return ret;
2455 }
2453 2456
2454 /* 2457 /*
2455 * Enable the device advanced error reporting. It's not critical to 2458 * Enable the device advanced error reporting. It's not critical to
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 4de074a86073..156b45cd4a19 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -74,12 +74,6 @@ MODULE_AUTHOR("Intel Corporation");
74#define bar0_off(base, bar) ((base) + ((bar) << 2)) 74#define bar0_off(base, bar) ((base) + ((bar) << 2))
75#define bar2_off(base, bar) bar0_off(base, (bar) - 2) 75#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
76 76
77static const struct intel_ntb_reg atom_reg;
78static const struct intel_ntb_alt_reg atom_pri_reg;
79static const struct intel_ntb_alt_reg atom_sec_reg;
80static const struct intel_ntb_alt_reg atom_b2b_reg;
81static const struct intel_ntb_xlat_reg atom_pri_xlat;
82static const struct intel_ntb_xlat_reg atom_sec_xlat;
83static const struct intel_ntb_reg xeon_reg; 77static const struct intel_ntb_reg xeon_reg;
84static const struct intel_ntb_alt_reg xeon_pri_reg; 78static const struct intel_ntb_alt_reg xeon_pri_reg;
85static const struct intel_ntb_alt_reg xeon_sec_reg; 79static const struct intel_ntb_alt_reg xeon_sec_reg;
@@ -184,15 +178,6 @@ static inline void _iowrite64(u64 val, void __iomem *mmio)
184#endif 178#endif
185#endif 179#endif
186 180
187static inline int pdev_is_atom(struct pci_dev *pdev)
188{
189 switch (pdev->device) {
190 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
191 return 1;
192 }
193 return 0;
194}
195
196static inline int pdev_is_xeon(struct pci_dev *pdev) 181static inline int pdev_is_xeon(struct pci_dev *pdev)
197{ 182{
198 switch (pdev->device) { 183 switch (pdev->device) {
@@ -1006,8 +991,7 @@ static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
1006{ 991{
1007 struct intel_ntb_dev *ndev = filp->private_data; 992 struct intel_ntb_dev *ndev = filp->private_data;
1008 993
1009 if (pdev_is_xeon(ndev->ntb.pdev) || 994 if (pdev_is_xeon(ndev->ntb.pdev))
1010 pdev_is_atom(ndev->ntb.pdev))
1011 return ndev_ntb_debugfs_read(filp, ubuf, count, offp); 995 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
1012 else if (pdev_is_skx_xeon(ndev->ntb.pdev)) 996 else if (pdev_is_skx_xeon(ndev->ntb.pdev))
1013 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp); 997 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
@@ -1439,242 +1423,6 @@ static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
1439 ndev->peer_reg->spad); 1423 ndev->peer_reg->spad);
1440} 1424}
1441 1425
1442/* ATOM */
1443
1444static u64 atom_db_ioread(void __iomem *mmio)
1445{
1446 return ioread64(mmio);
1447}
1448
1449static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1450{
1451 iowrite64(bits, mmio);
1452}
1453
1454static int atom_poll_link(struct intel_ntb_dev *ndev)
1455{
1456 u32 ntb_ctl;
1457
1458 ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1459
1460 if (ntb_ctl == ndev->ntb_ctl)
1461 return 0;
1462
1463 ndev->ntb_ctl = ntb_ctl;
1464
1465 ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1466
1467 return 1;
1468}
1469
1470static int atom_link_is_up(struct intel_ntb_dev *ndev)
1471{
1472 return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1473}
1474
1475static int atom_link_is_err(struct intel_ntb_dev *ndev)
1476{
1477 if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
1478 & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1479 return 1;
1480
1481 if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
1482 & ATOM_IBIST_ERR_OFLOW)
1483 return 1;
1484
1485 return 0;
1486}
1487
1488static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1489{
1490 struct device *dev = &ndev->ntb.pdev->dev;
1491
1492 switch (ppd & ATOM_PPD_TOPO_MASK) {
1493 case ATOM_PPD_TOPO_B2B_USD:
1494 dev_dbg(dev, "PPD %d B2B USD\n", ppd);
1495 return NTB_TOPO_B2B_USD;
1496
1497 case ATOM_PPD_TOPO_B2B_DSD:
1498 dev_dbg(dev, "PPD %d B2B DSD\n", ppd);
1499 return NTB_TOPO_B2B_DSD;
1500
1501 case ATOM_PPD_TOPO_PRI_USD:
1502 case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1503 case ATOM_PPD_TOPO_SEC_USD:
1504 case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1505 dev_dbg(dev, "PPD %d non B2B disabled\n", ppd);
1506 return NTB_TOPO_NONE;
1507 }
1508
1509 dev_dbg(dev, "PPD %d invalid\n", ppd);
1510 return NTB_TOPO_NONE;
1511}
1512
1513static void atom_link_hb(struct work_struct *work)
1514{
1515 struct intel_ntb_dev *ndev = hb_ndev(work);
1516 struct device *dev = &ndev->ntb.pdev->dev;
1517 unsigned long poll_ts;
1518 void __iomem *mmio;
1519 u32 status32;
1520
1521 poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1522
1523 /* Delay polling the link status if an interrupt was received,
1524 * unless the cached link status says the link is down.
1525 */
1526 if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1527 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1528 return;
1529 }
1530
1531 if (atom_poll_link(ndev))
1532 ntb_link_event(&ndev->ntb);
1533
1534 if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
1535 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1536 return;
1537 }
1538
1539 /* Link is down with error: recover the link! */
1540
1541 mmio = ndev->self_mmio;
1542
1543 /* Driver resets the NTB ModPhy lanes - magic! */
1544 iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
1545 iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
1546 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
1547 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1548
1549 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1550 msleep(100);
1551
1552 /* Clear AER Errors, write to clear */
1553 status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1554 dev_dbg(dev, "ERRCORSTS = %x\n", status32);
1555 status32 &= PCI_ERR_COR_REP_ROLL;
1556 iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1557
1558 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1559 status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1560 dev_dbg(dev, "LTSSMERRSTS0 = %x\n", status32);
1561 status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1562 iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1563
1564 /* Clear DeSkew Buffer error, write to clear */
1565 status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1566 dev_dbg(dev, "DESKEWSTS = %x\n", status32);
1567 status32 |= ATOM_DESKEWSTS_DBERR;
1568 iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1569
1570 status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1571 dev_dbg(dev, "IBSTERRRCRVSTS0 = %x\n", status32);
1572 status32 &= ATOM_IBIST_ERR_OFLOW;
1573 iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1574
1575 /* Releases the NTB state machine to allow the link to retrain */
1576 status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1577 dev_dbg(dev, "LTSSMSTATEJMP = %x\n", status32);
1578 status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1579 iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1580
1581 /* There is a potential race between the 2 NTB devices recovering at the
1582 * same time. If the times are the same, the link will not recover and
1583 * the driver will be stuck in this loop forever. Add a random interval
1584 * to the recovery time to prevent this race.
1585 */
1586 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
1587 + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1588}
1589
1590static int atom_init_isr(struct intel_ntb_dev *ndev)
1591{
1592 int rc;
1593
1594 rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
1595 ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1596 if (rc)
1597 return rc;
1598
1599 /* ATOM doesn't have link status interrupt, poll on that platform */
1600 ndev->last_ts = jiffies;
1601 INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
1602 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1603
1604 return 0;
1605}
1606
1607static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1608{
1609 cancel_delayed_work_sync(&ndev->hb_timer);
1610 ndev_deinit_isr(ndev);
1611}
1612
1613static int atom_init_ntb(struct intel_ntb_dev *ndev)
1614{
1615 ndev->mw_count = ATOM_MW_COUNT;
1616 ndev->spad_count = ATOM_SPAD_COUNT;
1617 ndev->db_count = ATOM_DB_COUNT;
1618
1619 switch (ndev->ntb.topo) {
1620 case NTB_TOPO_B2B_USD:
1621 case NTB_TOPO_B2B_DSD:
1622 ndev->self_reg = &atom_pri_reg;
1623 ndev->peer_reg = &atom_b2b_reg;
1624 ndev->xlat_reg = &atom_sec_xlat;
1625
1626 /* Enable Bus Master and Memory Space on the secondary side */
1627 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1628 ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1629
1630 break;
1631
1632 default:
1633 return -EINVAL;
1634 }
1635
1636 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1637
1638 return 0;
1639}
1640
1641static int atom_init_dev(struct intel_ntb_dev *ndev)
1642{
1643 u32 ppd;
1644 int rc;
1645
1646 rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1647 if (rc)
1648 return -EIO;
1649
1650 ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1651 if (ndev->ntb.topo == NTB_TOPO_NONE)
1652 return -EINVAL;
1653
1654 rc = atom_init_ntb(ndev);
1655 if (rc)
1656 return rc;
1657
1658 rc = atom_init_isr(ndev);
1659 if (rc)
1660 return rc;
1661
1662 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1663 /* Initiate PCI-E link training */
1664 rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
1665 ppd | ATOM_PPD_INIT_LINK);
1666 if (rc)
1667 return rc;
1668 }
1669
1670 return 0;
1671}
1672
1673static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1674{
1675 atom_deinit_isr(ndev);
1676}
1677
1678/* Skylake Xeon NTB */ 1426/* Skylake Xeon NTB */
1679 1427
1680static int skx_poll_link(struct intel_ntb_dev *ndev) 1428static int skx_poll_link(struct intel_ntb_dev *ndev)
@@ -2586,6 +2334,10 @@ static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
2586 goto err_dma_mask; 2334 goto err_dma_mask;
2587 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); 2335 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
2588 } 2336 }
2337 rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
2338 dma_get_mask(&pdev->dev));
2339 if (rc)
2340 goto err_dma_mask;
2589 2341
2590 ndev->self_mmio = pci_iomap(pdev, 0, 0); 2342 ndev->self_mmio = pci_iomap(pdev, 0, 0);
2591 if (!ndev->self_mmio) { 2343 if (!ndev->self_mmio) {
@@ -2658,24 +2410,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
2658 2410
2659 node = dev_to_node(&pdev->dev); 2411 node = dev_to_node(&pdev->dev);
2660 2412
2661 if (pdev_is_atom(pdev)) { 2413 if (pdev_is_xeon(pdev)) {
2662 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2663 if (!ndev) {
2664 rc = -ENOMEM;
2665 goto err_ndev;
2666 }
2667
2668 ndev_init_struct(ndev, pdev);
2669
2670 rc = intel_ntb_init_pci(ndev, pdev);
2671 if (rc)
2672 goto err_init_pci;
2673
2674 rc = atom_init_dev(ndev);
2675 if (rc)
2676 goto err_init_dev;
2677
2678 } else if (pdev_is_xeon(pdev)) {
2679 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 2414 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2680 if (!ndev) { 2415 if (!ndev) {
2681 rc = -ENOMEM; 2416 rc = -ENOMEM;
@@ -2731,9 +2466,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev,
2731 2466
2732err_register: 2467err_register:
2733 ndev_deinit_debugfs(ndev); 2468 ndev_deinit_debugfs(ndev);
2734 if (pdev_is_atom(pdev)) 2469 if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2735 atom_deinit_dev(ndev);
2736 else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2737 xeon_deinit_dev(ndev); 2470 xeon_deinit_dev(ndev);
2738err_init_dev: 2471err_init_dev:
2739 intel_ntb_deinit_pci(ndev); 2472 intel_ntb_deinit_pci(ndev);
@@ -2749,41 +2482,12 @@ static void intel_ntb_pci_remove(struct pci_dev *pdev)
2749 2482
2750 ntb_unregister_device(&ndev->ntb); 2483 ntb_unregister_device(&ndev->ntb);
2751 ndev_deinit_debugfs(ndev); 2484 ndev_deinit_debugfs(ndev);
2752 if (pdev_is_atom(pdev)) 2485 if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2753 atom_deinit_dev(ndev);
2754 else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2755 xeon_deinit_dev(ndev); 2486 xeon_deinit_dev(ndev);
2756 intel_ntb_deinit_pci(ndev); 2487 intel_ntb_deinit_pci(ndev);
2757 kfree(ndev); 2488 kfree(ndev);
2758} 2489}
2759 2490
2760static const struct intel_ntb_reg atom_reg = {
2761 .poll_link = atom_poll_link,
2762 .link_is_up = atom_link_is_up,
2763 .db_ioread = atom_db_ioread,
2764 .db_iowrite = atom_db_iowrite,
2765 .db_size = sizeof(u64),
2766 .ntb_ctl = ATOM_NTBCNTL_OFFSET,
2767 .mw_bar = {2, 4},
2768};
2769
2770static const struct intel_ntb_alt_reg atom_pri_reg = {
2771 .db_bell = ATOM_PDOORBELL_OFFSET,
2772 .db_mask = ATOM_PDBMSK_OFFSET,
2773 .spad = ATOM_SPAD_OFFSET,
2774};
2775
2776static const struct intel_ntb_alt_reg atom_b2b_reg = {
2777 .db_bell = ATOM_B2B_DOORBELL_OFFSET,
2778 .spad = ATOM_B2B_SPAD_OFFSET,
2779};
2780
2781static const struct intel_ntb_xlat_reg atom_sec_xlat = {
2782 /* FIXME : .bar0_base = ATOM_SBAR0BASE_OFFSET, */
2783 /* FIXME : .bar2_limit = ATOM_SBAR2LMT_OFFSET, */
2784 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
2785};
2786
2787static const struct intel_ntb_reg xeon_reg = { 2491static const struct intel_ntb_reg xeon_reg = {
2788 .poll_link = xeon_poll_link, 2492 .poll_link = xeon_poll_link,
2789 .link_is_up = xeon_link_is_up, 2493 .link_is_up = xeon_link_is_up,
@@ -2940,7 +2644,6 @@ static const struct file_operations intel_ntb_debugfs_info = {
2940}; 2644};
2941 2645
2942static const struct pci_device_id intel_ntb_pci_tbl[] = { 2646static const struct pci_device_id intel_ntb_pci_tbl[] = {
2943 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
2944 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, 2647 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2945 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, 2648 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2946 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, 2649 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h
index 2d6c38afb128..4415aa7ea775 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -66,7 +66,6 @@
66#define PCI_DEVICE_ID_INTEL_NTB_B2B_HSX 0x2F0D 66#define PCI_DEVICE_ID_INTEL_NTB_B2B_HSX 0x2F0D
67#define PCI_DEVICE_ID_INTEL_NTB_PS_HSX 0x2F0E 67#define PCI_DEVICE_ID_INTEL_NTB_PS_HSX 0x2F0E
68#define PCI_DEVICE_ID_INTEL_NTB_SS_HSX 0x2F0F 68#define PCI_DEVICE_ID_INTEL_NTB_SS_HSX 0x2F0F
69#define PCI_DEVICE_ID_INTEL_NTB_B2B_BWD 0x0C4E
70#define PCI_DEVICE_ID_INTEL_NTB_B2B_BDX 0x6F0D 69#define PCI_DEVICE_ID_INTEL_NTB_B2B_BDX 0x6F0D
71#define PCI_DEVICE_ID_INTEL_NTB_PS_BDX 0x6F0E 70#define PCI_DEVICE_ID_INTEL_NTB_PS_BDX 0x6F0E
72#define PCI_DEVICE_ID_INTEL_NTB_SS_BDX 0x6F0F 71#define PCI_DEVICE_ID_INTEL_NTB_SS_BDX 0x6F0F
@@ -196,63 +195,6 @@
196#define SKX_DB_TOTAL_SHIFT 33 195#define SKX_DB_TOTAL_SHIFT 33
197#define SKX_SPAD_COUNT 16 196#define SKX_SPAD_COUNT 16
198 197
199/* Intel Atom hardware */
200
201#define ATOM_SBAR2XLAT_OFFSET 0x0008
202#define ATOM_PDOORBELL_OFFSET 0x0020
203#define ATOM_PDBMSK_OFFSET 0x0028
204#define ATOM_NTBCNTL_OFFSET 0x0060
205#define ATOM_SPAD_OFFSET 0x0080
206#define ATOM_PPD_OFFSET 0x00d4
207#define ATOM_PBAR2XLAT_OFFSET 0x8008
208#define ATOM_B2B_DOORBELL_OFFSET 0x8020
209#define ATOM_B2B_SPAD_OFFSET 0x8080
210#define ATOM_SPCICMD_OFFSET 0xb004
211#define ATOM_LINK_STATUS_OFFSET 0xb052
212#define ATOM_ERRCORSTS_OFFSET 0xb110
213#define ATOM_IP_BASE 0xc000
214#define ATOM_DESKEWSTS_OFFSET (ATOM_IP_BASE + 0x3024)
215#define ATOM_LTSSMERRSTS0_OFFSET (ATOM_IP_BASE + 0x3180)
216#define ATOM_LTSSMSTATEJMP_OFFSET (ATOM_IP_BASE + 0x3040)
217#define ATOM_IBSTERRRCRVSTS0_OFFSET (ATOM_IP_BASE + 0x3324)
218#define ATOM_MODPHY_PCSREG4 0x1c004
219#define ATOM_MODPHY_PCSREG6 0x1c006
220
221#define ATOM_PPD_INIT_LINK 0x0008
222#define ATOM_PPD_CONN_MASK 0x0300
223#define ATOM_PPD_CONN_TRANSPARENT 0x0000
224#define ATOM_PPD_CONN_B2B 0x0100
225#define ATOM_PPD_CONN_RP 0x0200
226#define ATOM_PPD_DEV_MASK 0x1000
227#define ATOM_PPD_DEV_USD 0x0000
228#define ATOM_PPD_DEV_DSD 0x1000
229#define ATOM_PPD_TOPO_MASK (ATOM_PPD_CONN_MASK | ATOM_PPD_DEV_MASK)
230#define ATOM_PPD_TOPO_PRI_USD (ATOM_PPD_CONN_TRANSPARENT | ATOM_PPD_DEV_USD)
231#define ATOM_PPD_TOPO_PRI_DSD (ATOM_PPD_CONN_TRANSPARENT | ATOM_PPD_DEV_DSD)
232#define ATOM_PPD_TOPO_SEC_USD (ATOM_PPD_CONN_RP | ATOM_PPD_DEV_USD)
233#define ATOM_PPD_TOPO_SEC_DSD (ATOM_PPD_CONN_RP | ATOM_PPD_DEV_DSD)
234#define ATOM_PPD_TOPO_B2B_USD (ATOM_PPD_CONN_B2B | ATOM_PPD_DEV_USD)
235#define ATOM_PPD_TOPO_B2B_DSD (ATOM_PPD_CONN_B2B | ATOM_PPD_DEV_DSD)
236
237#define ATOM_MW_COUNT 2
238#define ATOM_DB_COUNT 34
239#define ATOM_DB_VALID_MASK (BIT_ULL(ATOM_DB_COUNT) - 1)
240#define ATOM_DB_MSIX_VECTOR_COUNT 34
241#define ATOM_DB_MSIX_VECTOR_SHIFT 1
242#define ATOM_DB_TOTAL_SHIFT 34
243#define ATOM_SPAD_COUNT 16
244
245#define ATOM_NTB_CTL_DOWN_BIT BIT(16)
246#define ATOM_NTB_CTL_ACTIVE(x) !(x & ATOM_NTB_CTL_DOWN_BIT)
247
248#define ATOM_DESKEWSTS_DBERR BIT(15)
249#define ATOM_LTSSMERRSTS0_UNEXPECTEDEI BIT(20)
250#define ATOM_LTSSMSTATEJMP_FORCEDETECT BIT(2)
251#define ATOM_IBIST_ERR_OFLOW 0x7FFF7FFF
252
253#define ATOM_LINK_HB_TIMEOUT msecs_to_jiffies(1000)
254#define ATOM_LINK_RECOVERY_TIME msecs_to_jiffies(500)
255
256/* Ntb control and link status */ 198/* Ntb control and link status */
257 199
258#define NTB_CTL_CFG_LOCK BIT(0) 200#define NTB_CTL_CFG_LOCK BIT(0)
diff --git a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
index afe8ed6f3b23..f624ae27eabe 100644
--- a/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
+++ b/drivers/ntb/hw/mscc/ntb_hw_switchtec.c
@@ -94,6 +94,9 @@ struct switchtec_ntb {
94 struct ntb_ctrl_regs __iomem *mmio_self_ctrl; 94 struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
95 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; 95 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
96 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; 96 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
97 struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
98
99 void __iomem *mmio_xlink_win;
97 100
98 struct shared_mw *self_shared; 101 struct shared_mw *self_shared;
99 struct shared_mw __iomem *peer_shared; 102 struct shared_mw __iomem *peer_shared;
@@ -109,6 +112,7 @@ struct switchtec_ntb {
109 112
110 int nr_direct_mw; 113 int nr_direct_mw;
111 int nr_lut_mw; 114 int nr_lut_mw;
115 int nr_rsvd_luts;
112 int direct_mw_to_bar[MAX_DIRECT_MW]; 116 int direct_mw_to_bar[MAX_DIRECT_MW];
113 117
114 int peer_nr_direct_mw; 118 int peer_nr_direct_mw;
@@ -118,6 +122,7 @@ struct switchtec_ntb {
118 bool link_is_up; 122 bool link_is_up;
119 enum ntb_speed link_speed; 123 enum ntb_speed link_speed;
120 enum ntb_width link_width; 124 enum ntb_width link_width;
125 struct work_struct link_reinit_work;
121}; 126};
122 127
123static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) 128static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
@@ -172,7 +177,7 @@ static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
172 177
173 if (ps == status) { 178 if (ps == status) {
174 dev_err(&sndev->stdev->dev, 179 dev_err(&sndev->stdev->dev,
175 "Timed out while peforming %s (%d). (%08x)", 180 "Timed out while performing %s (%d). (%08x)\n",
176 op_text[op], op, 181 op_text[op], op,
177 ioread32(&ctl->partition_status)); 182 ioread32(&ctl->partition_status));
178 183
@@ -185,10 +190,10 @@ static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
185static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, 190static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
186 u32 val) 191 u32 val)
187{ 192{
188 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_self_dbmsg->omsg)) 193 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
189 return -EINVAL; 194 return -EINVAL;
190 195
191 iowrite32(val, &sndev->mmio_self_dbmsg->omsg[idx].msg); 196 iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
192 197
193 return 0; 198 return 0;
194} 199}
@@ -197,7 +202,7 @@ static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
197{ 202{
198 struct switchtec_ntb *sndev = ntb_sndev(ntb); 203 struct switchtec_ntb *sndev = ntb_sndev(ntb);
199 int nr_direct_mw = sndev->peer_nr_direct_mw; 204 int nr_direct_mw = sndev->peer_nr_direct_mw;
200 int nr_lut_mw = sndev->peer_nr_lut_mw - 1; 205 int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
201 206
202 if (pidx != NTB_DEF_PEER_IDX) 207 if (pidx != NTB_DEF_PEER_IDX)
203 return -EINVAL; 208 return -EINVAL;
@@ -210,12 +215,12 @@ static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
210 215
211static int lut_index(struct switchtec_ntb *sndev, int mw_idx) 216static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
212{ 217{
213 return mw_idx - sndev->nr_direct_mw + 1; 218 return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
214} 219}
215 220
216static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx) 221static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
217{ 222{
218 return mw_idx - sndev->peer_nr_direct_mw + 1; 223 return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
219} 224}
220 225
221static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, 226static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
@@ -306,7 +311,7 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
306 if (pidx != NTB_DEF_PEER_IDX) 311 if (pidx != NTB_DEF_PEER_IDX)
307 return -EINVAL; 312 return -EINVAL;
308 313
309 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap", 314 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
310 widx, pidx, &addr, &size); 315 widx, pidx, &addr, &size);
311 316
312 if (widx >= switchtec_ntb_mw_count(ntb, pidx)) 317 if (widx >= switchtec_ntb_mw_count(ntb, pidx))
@@ -315,6 +320,19 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
315 if (xlate_pos < 12) 320 if (xlate_pos < 12)
316 return -EINVAL; 321 return -EINVAL;
317 322
323 if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
324 /*
325 * In certain circumstances we can get a buffer that is
326 * not aligned to its size. (Most of the time
327 * dma_alloc_coherent ensures this). This can happen when
328 * using large buffers allocated by the CMA
329 * (see CMA_CONFIG_ALIGNMENT)
330 */
331 dev_err(&sndev->stdev->dev,
332 "ERROR: Memory window address is not aligned to it's size!\n");
333 return -EINVAL;
334 }
335
318 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 336 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
319 NTB_CTRL_PART_STATUS_LOCKED); 337 NTB_CTRL_PART_STATUS_LOCKED);
320 if (rc) 338 if (rc)
@@ -337,7 +355,7 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
337 355
338 if (rc == -EIO) { 356 if (rc == -EIO) {
339 dev_err(&sndev->stdev->dev, 357 dev_err(&sndev->stdev->dev,
340 "Hardware reported an error configuring mw %d: %08x", 358 "Hardware reported an error configuring mw %d: %08x\n",
341 widx, ioread32(&ctl->bar_error)); 359 widx, ioread32(&ctl->bar_error));
342 360
343 if (widx < nr_direct_mw) 361 if (widx < nr_direct_mw)
@@ -355,8 +373,9 @@ static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
355static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb) 373static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
356{ 374{
357 struct switchtec_ntb *sndev = ntb_sndev(ntb); 375 struct switchtec_ntb *sndev = ntb_sndev(ntb);
376 int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
358 377
359 return sndev->nr_direct_mw + (use_lut_mws ? sndev->nr_lut_mw - 1 : 0); 378 return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
360} 379}
361 380
362static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev, 381static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
@@ -463,18 +482,69 @@ static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
463 sndev->link_width = min(self_width, peer_width); 482 sndev->link_width = min(self_width, peer_width);
464} 483}
465 484
466enum { 485static int crosslink_is_enabled(struct switchtec_ntb *sndev)
486{
487 struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
488
489 return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
490}
491
492static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
493{
494 int i;
495 u32 msg_map = 0;
496
497 if (!crosslink_is_enabled(sndev))
498 return;
499
500 for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
501 int m = i | sndev->self_partition << 2;
502
503 msg_map |= m << i * 8;
504 }
505
506 iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
507 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
508 &sndev->mmio_peer_dbmsg->odb_mask);
509}
510
511enum switchtec_msg {
467 LINK_MESSAGE = 0, 512 LINK_MESSAGE = 0,
468 MSG_LINK_UP = 1, 513 MSG_LINK_UP = 1,
469 MSG_LINK_DOWN = 2, 514 MSG_LINK_DOWN = 2,
470 MSG_CHECK_LINK = 3, 515 MSG_CHECK_LINK = 3,
516 MSG_LINK_FORCE_DOWN = 4,
471}; 517};
472 518
473static void switchtec_ntb_check_link(struct switchtec_ntb *sndev) 519static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
520
521static void link_reinit_work(struct work_struct *work)
522{
523 struct switchtec_ntb *sndev;
524
525 sndev = container_of(work, struct switchtec_ntb, link_reinit_work);
526
527 switchtec_ntb_reinit_peer(sndev);
528}
529
530static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
531 enum switchtec_msg msg)
474{ 532{
475 int link_sta; 533 int link_sta;
476 int old = sndev->link_is_up; 534 int old = sndev->link_is_up;
477 535
536 if (msg == MSG_LINK_FORCE_DOWN) {
537 schedule_work(&sndev->link_reinit_work);
538
539 if (sndev->link_is_up) {
540 sndev->link_is_up = 0;
541 ntb_link_event(&sndev->ntb);
542 dev_info(&sndev->stdev->dev, "ntb link forced down\n");
543 }
544
545 return;
546 }
547
478 link_sta = sndev->self_shared->link_sta; 548 link_sta = sndev->self_shared->link_sta;
479 if (link_sta) { 549 if (link_sta) {
480 u64 peer = ioread64(&sndev->peer_shared->magic); 550 u64 peer = ioread64(&sndev->peer_shared->magic);
@@ -491,8 +561,11 @@ static void switchtec_ntb_check_link(struct switchtec_ntb *sndev)
491 if (link_sta != old) { 561 if (link_sta != old) {
492 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); 562 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
493 ntb_link_event(&sndev->ntb); 563 ntb_link_event(&sndev->ntb);
494 dev_info(&sndev->stdev->dev, "ntb link %s", 564 dev_info(&sndev->stdev->dev, "ntb link %s\n",
495 link_sta ? "up" : "down"); 565 link_sta ? "up" : "down");
566
567 if (link_sta)
568 crosslink_init_dbmsgs(sndev);
496 } 569 }
497} 570}
498 571
@@ -500,7 +573,7 @@ static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
500{ 573{
501 struct switchtec_ntb *sndev = stdev->sndev; 574 struct switchtec_ntb *sndev = stdev->sndev;
502 575
503 switchtec_ntb_check_link(sndev); 576 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
504} 577}
505 578
506static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, 579static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
@@ -523,12 +596,12 @@ static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
523{ 596{
524 struct switchtec_ntb *sndev = ntb_sndev(ntb); 597 struct switchtec_ntb *sndev = ntb_sndev(ntb);
525 598
526 dev_dbg(&sndev->stdev->dev, "enabling link"); 599 dev_dbg(&sndev->stdev->dev, "enabling link\n");
527 600
528 sndev->self_shared->link_sta = 1; 601 sndev->self_shared->link_sta = 1;
529 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); 602 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
530 603
531 switchtec_ntb_check_link(sndev); 604 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
532 605
533 return 0; 606 return 0;
534} 607}
@@ -537,12 +610,12 @@ static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
537{ 610{
538 struct switchtec_ntb *sndev = ntb_sndev(ntb); 611 struct switchtec_ntb *sndev = ntb_sndev(ntb);
539 612
540 dev_dbg(&sndev->stdev->dev, "disabling link"); 613 dev_dbg(&sndev->stdev->dev, "disabling link\n");
541 614
542 sndev->self_shared->link_sta = 0; 615 sndev->self_shared->link_sta = 0;
543 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); 616 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
544 617
545 switchtec_ntb_check_link(sndev); 618 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
546 619
547 return 0; 620 return 0;
548} 621}
@@ -638,7 +711,7 @@ static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
638 struct switchtec_ntb *sndev = ntb_sndev(ntb); 711 struct switchtec_ntb *sndev = ntb_sndev(ntb);
639 unsigned long offset; 712 unsigned long offset;
640 713
641 offset = (unsigned long)sndev->mmio_self_dbmsg->odb - 714 offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
642 (unsigned long)sndev->stdev->mmio; 715 (unsigned long)sndev->stdev->mmio;
643 716
644 offset += sndev->db_shift / 8; 717 offset += sndev->db_shift / 8;
@@ -656,7 +729,7 @@ static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
656 struct switchtec_ntb *sndev = ntb_sndev(ntb); 729 struct switchtec_ntb *sndev = ntb_sndev(ntb);
657 730
658 iowrite64(db_bits << sndev->db_peer_shift, 731 iowrite64(db_bits << sndev->db_peer_shift,
659 &sndev->mmio_self_dbmsg->odb); 732 &sndev->mmio_peer_dbmsg->odb);
660 733
661 return 0; 734 return 0;
662} 735}
@@ -777,24 +850,63 @@ static const struct ntb_dev_ops switchtec_ntb_ops = {
777 .peer_spad_addr = switchtec_ntb_peer_spad_addr, 850 .peer_spad_addr = switchtec_ntb_peer_spad_addr,
778}; 851};
779 852
780static void switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) 853static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
781{ 854{
855 u64 tpart_vec;
856 int self;
782 u64 part_map; 857 u64 part_map;
858 int bit;
783 859
784 sndev->ntb.pdev = sndev->stdev->pdev; 860 sndev->ntb.pdev = sndev->stdev->pdev;
785 sndev->ntb.topo = NTB_TOPO_SWITCH; 861 sndev->ntb.topo = NTB_TOPO_SWITCH;
786 sndev->ntb.ops = &switchtec_ntb_ops; 862 sndev->ntb.ops = &switchtec_ntb_ops;
787 863
864 INIT_WORK(&sndev->link_reinit_work, link_reinit_work);
865
788 sndev->self_partition = sndev->stdev->partition; 866 sndev->self_partition = sndev->stdev->partition;
789 867
790 sndev->mmio_ntb = sndev->stdev->mmio_ntb; 868 sndev->mmio_ntb = sndev->stdev->mmio_ntb;
869
870 self = sndev->self_partition;
871 tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
872 tpart_vec <<= 32;
873 tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
874
791 part_map = ioread64(&sndev->mmio_ntb->ep_map); 875 part_map = ioread64(&sndev->mmio_ntb->ep_map);
792 part_map &= ~(1 << sndev->self_partition); 876 part_map &= ~(1 << sndev->self_partition);
793 sndev->peer_partition = ffs(part_map) - 1;
794 877
795 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d (%llx)", 878 if (!ffs(tpart_vec)) {
796 sndev->self_partition, sndev->stdev->partition_count, 879 if (sndev->stdev->partition_count != 2) {
797 part_map); 880 dev_err(&sndev->stdev->dev,
881 "ntb target partition not defined\n");
882 return -ENODEV;
883 }
884
885 bit = ffs(part_map);
886 if (!bit) {
887 dev_err(&sndev->stdev->dev,
888 "peer partition is not NT partition\n");
889 return -ENODEV;
890 }
891
892 sndev->peer_partition = bit - 1;
893 } else {
894 if (ffs(tpart_vec) != fls(tpart_vec)) {
895 dev_err(&sndev->stdev->dev,
896 "ntb driver only supports 1 pair of 1-1 ntb mapping\n");
897 return -ENODEV;
898 }
899
900 sndev->peer_partition = ffs(tpart_vec) - 1;
901 if (!(part_map & (1 << sndev->peer_partition))) {
902 dev_err(&sndev->stdev->dev,
903 "ntb target partition is not NT partition\n");
904 return -ENODEV;
905 }
906 }
907
908 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
909 sndev->self_partition, sndev->stdev->partition_count);
798 910
799 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + 911 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
800 SWITCHTEC_NTB_REG_CTRL_OFFSET; 912 SWITCHTEC_NTB_REG_CTRL_OFFSET;
@@ -804,6 +916,283 @@ static void switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
804 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; 916 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
805 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; 917 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
806 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; 918 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
919 sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
920
921 return 0;
922}
923
924static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
925 struct ntb_ctrl_regs __iomem *ctl,
926 int lut_idx, int partition, u64 addr)
927{
928 int peer_bar = sndev->peer_direct_mw_to_bar[0];
929 u32 ctl_val;
930 int rc;
931
932 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
933 NTB_CTRL_PART_STATUS_LOCKED);
934 if (rc)
935 return rc;
936
937 ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
938 ctl_val &= 0xFF;
939 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
940 ctl_val |= ilog2(LUT_SIZE) << 8;
941 ctl_val |= (sndev->nr_lut_mw - 1) << 14;
942 iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
943
944 iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
945 &ctl->lut_entry[lut_idx]);
946
947 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
948 NTB_CTRL_PART_STATUS_NORMAL);
949 if (rc) {
950 u32 bar_error, lut_error;
951
952 bar_error = ioread32(&ctl->bar_error);
953 lut_error = ioread32(&ctl->lut_error);
954 dev_err(&sndev->stdev->dev,
955 "Error setting up reserved lut window: %08x / %08x\n",
956 bar_error, lut_error);
957 return rc;
958 }
959
960 return 0;
961}
962
963static int config_req_id_table(struct switchtec_ntb *sndev,
964 struct ntb_ctrl_regs __iomem *mmio_ctrl,
965 int *req_ids, int count)
966{
967 int i, rc = 0;
968 u32 error;
969 u32 proxy_id;
970
971 if (ioread32(&mmio_ctrl->req_id_table_size) < count) {
972 dev_err(&sndev->stdev->dev,
973 "Not enough requester IDs available.\n");
974 return -EFAULT;
975 }
976
977 rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
978 NTB_CTRL_PART_OP_LOCK,
979 NTB_CTRL_PART_STATUS_LOCKED);
980 if (rc)
981 return rc;
982
983 iowrite32(NTB_PART_CTRL_ID_PROT_DIS,
984 &mmio_ctrl->partition_ctrl);
985
986 for (i = 0; i < count; i++) {
987 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
988 &mmio_ctrl->req_id_table[i]);
989
990 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
991 dev_dbg(&sndev->stdev->dev,
992 "Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
993 req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
994 req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
995 (proxy_id >> 1) & 0x7);
996 }
997
998 rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
999 NTB_CTRL_PART_OP_CFG,
1000 NTB_CTRL_PART_STATUS_NORMAL);
1001
1002 if (rc == -EIO) {
1003 error = ioread32(&mmio_ctrl->req_id_error);
1004 dev_err(&sndev->stdev->dev,
1005 "Error setting up the requester ID table: %08x\n",
1006 error);
1007 }
1008
1009 return 0;
1010}
1011
1012static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
1013 u64 *mw_addrs, int mw_count)
1014{
1015 int rc, i;
1016 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
1017 u64 addr;
1018 size_t size, offset;
1019 int bar;
1020 int xlate_pos;
1021 u32 ctl_val;
1022
1023 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
1024 NTB_CTRL_PART_STATUS_LOCKED);
1025 if (rc)
1026 return rc;
1027
1028 for (i = 0; i < sndev->nr_lut_mw; i++) {
1029 if (i == ntb_lut_idx)
1030 continue;
1031
1032 addr = mw_addrs[0] + LUT_SIZE * i;
1033
1034 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1035 addr),
1036 &ctl->lut_entry[i]);
1037 }
1038
1039 sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1040
1041 for (i = 0; i < sndev->nr_direct_mw; i++) {
1042 bar = sndev->direct_mw_to_bar[i];
1043 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1044 addr = mw_addrs[i] + offset;
1045 size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1046 xlate_pos = ilog2(size);
1047
1048 if (offset && size > offset)
1049 size = offset;
1050
1051 ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1052 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1053
1054 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1055 iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size);
1056 iowrite64(sndev->peer_partition | addr,
1057 &ctl->bar_entry[bar].xlate_addr);
1058 }
1059
1060 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
1061 NTB_CTRL_PART_STATUS_NORMAL);
1062 if (rc) {
1063 u32 bar_error, lut_error;
1064
1065 bar_error = ioread32(&ctl->bar_error);
1066 lut_error = ioread32(&ctl->lut_error);
1067 dev_err(&sndev->stdev->dev,
1068 "Error setting up cross link windows: %08x / %08x\n",
1069 bar_error, lut_error);
1070 return rc;
1071 }
1072
1073 return 0;
1074}
1075
1076static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1077 struct ntb_ctrl_regs __iomem *mmio_ctrl)
1078{
1079 int req_ids[16];
1080 int i;
1081 u32 proxy_id;
1082
1083 for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1084 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1085
1086 if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1087 break;
1088
1089 req_ids[i] = ((proxy_id >> 1) & 0xFF);
1090 }
1091
1092 return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
1093}
1094
1095/*
1096 * In crosslink configuration there is a virtual partition in the
1097 * middle of the two switches. The BARs in this partition have to be
1098 * enumerated and assigned addresses.
1099 */
1100static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1101 u64 *bar_addrs)
1102{
1103 struct part_cfg_regs __iomem *part_cfg =
1104 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1105 u32 pff = ioread32(&part_cfg->vep_pff_inst_id);
1106 struct pff_csr_regs __iomem *mmio_pff =
1107 &sndev->stdev->mmio_pff_csr[pff];
1108 const u64 bar_space = 0x1000000000LL;
1109 u64 bar_addr;
1110 int bar_cnt = 0;
1111 int i;
1112
1113 iowrite16(0x6, &mmio_pff->pcicmd);
1114
1115 for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1116 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
1117 bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
1118 bar_addr &= ~0xf;
1119
1120 dev_dbg(&sndev->stdev->dev,
1121 "Crosslink BAR%d addr: %llx\n",
1122 i, bar_addr);
1123
1124 if (bar_addr != bar_space * i)
1125 continue;
1126
1127 bar_addrs[bar_cnt++] = bar_addr;
1128 }
1129
1130 return bar_cnt;
1131}
1132
1133static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1134{
1135 int rc;
1136 int bar = sndev->direct_mw_to_bar[0];
1137 const int ntb_lut_idx = 1;
1138 u64 bar_addrs[6];
1139 u64 addr;
1140 int offset;
1141 int bar_cnt;
1142
1143 if (!crosslink_is_enabled(sndev))
1144 return 0;
1145
1146 dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1147 sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1148
1149 bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1150 if (bar_cnt < sndev->nr_direct_mw + 1) {
1151 dev_err(&sndev->stdev->dev,
1152 "Error enumerating crosslink partition\n");
1153 return -EINVAL;
1154 }
1155
1156 addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1157 SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1158 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1159
1160 offset = addr & (LUT_SIZE - 1);
1161 addr -= offset;
1162
1163 rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
1164 sndev->peer_partition, addr);
1165 if (rc)
1166 return rc;
1167
1168 rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
1169 bar_cnt - 1);
1170 if (rc)
1171 return rc;
1172
1173 rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
1174 if (rc)
1175 return rc;
1176
1177 sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
1178 LUT_SIZE, LUT_SIZE);
1179 if (!sndev->mmio_xlink_win) {
1180 rc = -ENOMEM;
1181 return rc;
1182 }
1183
1184 sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1185 sndev->nr_rsvd_luts++;
1186
1187 crosslink_init_dbmsgs(sndev);
1188
1189 return 0;
1190}
1191
1192static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1193{
1194 if (sndev->mmio_xlink_win)
1195 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
807} 1196}
808 1197
809static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) 1198static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
@@ -829,7 +1218,7 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
829 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); 1218 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
830 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); 1219 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
831 1220
832 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut", 1221 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
833 sndev->nr_direct_mw, sndev->nr_lut_mw); 1222 sndev->nr_direct_mw, sndev->nr_lut_mw);
834 1223
835 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, 1224 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
@@ -839,7 +1228,7 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
839 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); 1228 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
840 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); 1229 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
841 1230
842 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut", 1231 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
843 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); 1232 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
844 1233
845} 1234}
@@ -849,24 +1238,35 @@ static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
849 * shared among all partitions. So we must split them in half 1238 * shared among all partitions. So we must split them in half
850 * (32 for each partition). However, the message interrupts are 1239 * (32 for each partition). However, the message interrupts are
851 * also shared with the top 4 doorbells so we just limit this to 1240 * also shared with the top 4 doorbells so we just limit this to
852 * 28 doorbells per partition 1241 * 28 doorbells per partition.
1242 *
1243 * In crosslink mode, each side has it's own dbmsg register so
1244 * they can each use all 60 of the available doorbells.
853 */ 1245 */
854static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) 1246static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
855{ 1247{
856 sndev->db_valid_mask = 0x0FFFFFFF; 1248 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
857 1249
858 if (sndev->self_partition < sndev->peer_partition) { 1250 if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1251 sndev->db_shift = 0;
1252 sndev->db_peer_shift = 0;
1253 sndev->db_valid_mask = sndev->db_mask;
1254 } else if (sndev->self_partition < sndev->peer_partition) {
859 sndev->db_shift = 0; 1255 sndev->db_shift = 0;
860 sndev->db_peer_shift = 32; 1256 sndev->db_peer_shift = 32;
1257 sndev->db_valid_mask = 0x0FFFFFFF;
861 } else { 1258 } else {
862 sndev->db_shift = 32; 1259 sndev->db_shift = 32;
863 sndev->db_peer_shift = 0; 1260 sndev->db_peer_shift = 0;
1261 sndev->db_valid_mask = 0x0FFFFFFF;
864 } 1262 }
865 1263
866 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
867 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 1264 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
868 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 1265 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
869 &sndev->mmio_self_dbmsg->odb_mask); 1266 &sndev->mmio_peer_dbmsg->odb_mask);
1267
1268 dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1269 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
870} 1270}
871 1271
872static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) 1272static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
@@ -887,52 +1287,23 @@ static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
887 &sndev->mmio_self_dbmsg->imsg[i]); 1287 &sndev->mmio_self_dbmsg->imsg[i]);
888} 1288}
889 1289
890static int switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) 1290static int
1291switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
891{ 1292{
892 int rc = 0; 1293 int req_ids[2];
893 u16 req_id;
894 u32 error;
895
896 req_id = ioread16(&sndev->mmio_ntb->requester_id);
897
898 if (ioread32(&sndev->mmio_self_ctrl->req_id_table_size) < 2) {
899 dev_err(&sndev->stdev->dev,
900 "Not enough requester IDs available.");
901 return -EFAULT;
902 }
903
904 rc = switchtec_ntb_part_op(sndev, sndev->mmio_self_ctrl,
905 NTB_CTRL_PART_OP_LOCK,
906 NTB_CTRL_PART_STATUS_LOCKED);
907 if (rc)
908 return rc;
909
910 iowrite32(NTB_PART_CTRL_ID_PROT_DIS,
911 &sndev->mmio_self_ctrl->partition_ctrl);
912 1294
913 /* 1295 /*
914 * Root Complex Requester ID (which is 0:00.0) 1296 * Root Complex Requester ID (which is 0:00.0)
915 */ 1297 */
916 iowrite32(0 << 16 | NTB_CTRL_REQ_ID_EN, 1298 req_ids[0] = 0;
917 &sndev->mmio_self_ctrl->req_id_table[0]);
918 1299
919 /* 1300 /*
920 * Host Bridge Requester ID (as read from the mmap address) 1301 * Host Bridge Requester ID (as read from the mmap address)
921 */ 1302 */
922 iowrite32(req_id << 16 | NTB_CTRL_REQ_ID_EN, 1303 req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
923 &sndev->mmio_self_ctrl->req_id_table[1]);
924
925 rc = switchtec_ntb_part_op(sndev, sndev->mmio_self_ctrl,
926 NTB_CTRL_PART_OP_CFG,
927 NTB_CTRL_PART_STATUS_NORMAL);
928 if (rc == -EIO) {
929 error = ioread32(&sndev->mmio_self_ctrl->req_id_error);
930 dev_err(&sndev->stdev->dev,
931 "Error setting up the requester ID table: %08x",
932 error);
933 }
934 1304
935 return rc; 1305 return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
1306 ARRAY_SIZE(req_ids));
936} 1307}
937 1308
938static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) 1309static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
@@ -963,59 +1334,35 @@ static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
963 1334
964static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) 1335static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
965{ 1336{
966 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 1337 int self_bar = sndev->direct_mw_to_bar[0];
967 int bar = sndev->direct_mw_to_bar[0];
968 u32 ctl_val;
969 int rc; 1338 int rc;
970 1339
1340 sndev->nr_rsvd_luts++;
971 sndev->self_shared = dma_zalloc_coherent(&sndev->stdev->pdev->dev, 1341 sndev->self_shared = dma_zalloc_coherent(&sndev->stdev->pdev->dev,
972 LUT_SIZE, 1342 LUT_SIZE,
973 &sndev->self_shared_dma, 1343 &sndev->self_shared_dma,
974 GFP_KERNEL); 1344 GFP_KERNEL);
975 if (!sndev->self_shared) { 1345 if (!sndev->self_shared) {
976 dev_err(&sndev->stdev->dev, 1346 dev_err(&sndev->stdev->dev,
977 "unable to allocate memory for shared mw"); 1347 "unable to allocate memory for shared mw\n");
978 return -ENOMEM; 1348 return -ENOMEM;
979 } 1349 }
980 1350
981 switchtec_ntb_init_shared(sndev); 1351 switchtec_ntb_init_shared(sndev);
982 1352
983 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 1353 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
984 NTB_CTRL_PART_STATUS_LOCKED); 1354 sndev->self_partition,
1355 sndev->self_shared_dma);
985 if (rc) 1356 if (rc)
986 goto unalloc_and_exit; 1357 goto unalloc_and_exit;
987 1358
988 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 1359 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
989 ctl_val &= 0xFF;
990 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
991 ctl_val |= ilog2(LUT_SIZE) << 8;
992 ctl_val |= (sndev->nr_lut_mw - 1) << 14;
993 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
994
995 iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) |
996 sndev->self_shared_dma),
997 &ctl->lut_entry[0]);
998
999 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
1000 NTB_CTRL_PART_STATUS_NORMAL);
1001 if (rc) {
1002 u32 bar_error, lut_error;
1003
1004 bar_error = ioread32(&ctl->bar_error);
1005 lut_error = ioread32(&ctl->lut_error);
1006 dev_err(&sndev->stdev->dev,
1007 "Error setting up shared MW: %08x / %08x",
1008 bar_error, lut_error);
1009 goto unalloc_and_exit;
1010 }
1011
1012 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, bar, LUT_SIZE);
1013 if (!sndev->peer_shared) { 1360 if (!sndev->peer_shared) {
1014 rc = -ENOMEM; 1361 rc = -ENOMEM;
1015 goto unalloc_and_exit; 1362 goto unalloc_and_exit;
1016 } 1363 }
1017 1364
1018 dev_dbg(&sndev->stdev->dev, "Shared MW Ready"); 1365 dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1019 return 0; 1366 return 0;
1020 1367
1021unalloc_and_exit: 1368unalloc_and_exit:
@@ -1034,6 +1381,7 @@ static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1034 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1381 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1035 sndev->self_shared, 1382 sndev->self_shared,
1036 sndev->self_shared_dma); 1383 sndev->self_shared_dma);
1384 sndev->nr_rsvd_luts--;
1037} 1385}
1038 1386
1039static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) 1387static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
@@ -1056,12 +1404,12 @@ static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1056 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); 1404 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
1057 1405
1058 if (msg & NTB_DBMSG_IMSG_STATUS) { 1406 if (msg & NTB_DBMSG_IMSG_STATUS) {
1059 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", i, 1407 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1060 (u32)msg); 1408 i, (u32)msg);
1061 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); 1409 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1062 1410
1063 if (i == LINK_MESSAGE) 1411 if (i == LINK_MESSAGE)
1064 switchtec_ntb_check_link(sndev); 1412 switchtec_ntb_check_link(sndev, msg);
1065 } 1413 }
1066 } 1414 }
1067 1415
@@ -1085,7 +1433,7 @@ static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1085 message_irq == event_irq) 1433 message_irq == event_irq)
1086 message_irq++; 1434 message_irq++;
1087 1435
1088 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d", 1436 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1089 event_irq, doorbell_irq, message_irq); 1437 event_irq, doorbell_irq, message_irq);
1090 1438
1091 for (i = 0; i < idb_vecs - 4; i++) 1439 for (i = 0; i < idb_vecs - 4; i++)
@@ -1122,6 +1470,14 @@ static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1122 free_irq(sndev->message_irq, sndev); 1470 free_irq(sndev->message_irq, sndev);
1123} 1471}
1124 1472
1473static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1474{
1475 dev_info(&sndev->stdev->dev, "peer reinitialized\n");
1476 switchtec_ntb_deinit_shared_mw(sndev);
1477 switchtec_ntb_init_mw(sndev);
1478 return switchtec_ntb_init_shared_mw(sndev);
1479}
1480
1125static int switchtec_ntb_add(struct device *dev, 1481static int switchtec_ntb_add(struct device *dev,
1126 struct class_interface *class_intf) 1482 struct class_interface *class_intf)
1127{ 1483{
@@ -1134,38 +1490,50 @@ static int switchtec_ntb_add(struct device *dev,
1134 if (stdev->pdev->class != MICROSEMI_NTB_CLASSCODE) 1490 if (stdev->pdev->class != MICROSEMI_NTB_CLASSCODE)
1135 return -ENODEV; 1491 return -ENODEV;
1136 1492
1137 if (stdev->partition_count != 2)
1138 dev_warn(dev, "ntb driver only supports 2 partitions");
1139
1140 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); 1493 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
1141 if (!sndev) 1494 if (!sndev)
1142 return -ENOMEM; 1495 return -ENOMEM;
1143 1496
1144 sndev->stdev = stdev; 1497 sndev->stdev = stdev;
1145 switchtec_ntb_init_sndev(sndev); 1498 rc = switchtec_ntb_init_sndev(sndev);
1499 if (rc)
1500 goto free_and_exit;
1501
1146 switchtec_ntb_init_mw(sndev); 1502 switchtec_ntb_init_mw(sndev);
1147 switchtec_ntb_init_db(sndev);
1148 switchtec_ntb_init_msgs(sndev);
1149 1503
1150 rc = switchtec_ntb_init_req_id_table(sndev); 1504 rc = switchtec_ntb_init_req_id_table(sndev);
1151 if (rc) 1505 if (rc)
1152 goto free_and_exit; 1506 goto free_and_exit;
1153 1507
1154 rc = switchtec_ntb_init_shared_mw(sndev); 1508 rc = switchtec_ntb_init_crosslink(sndev);
1155 if (rc) 1509 if (rc)
1156 goto free_and_exit; 1510 goto free_and_exit;
1157 1511
1512 switchtec_ntb_init_db(sndev);
1513 switchtec_ntb_init_msgs(sndev);
1514
1515 rc = switchtec_ntb_init_shared_mw(sndev);
1516 if (rc)
1517 goto deinit_crosslink;
1518
1158 rc = switchtec_ntb_init_db_msg_irq(sndev); 1519 rc = switchtec_ntb_init_db_msg_irq(sndev);
1159 if (rc) 1520 if (rc)
1160 goto deinit_shared_and_exit; 1521 goto deinit_shared_and_exit;
1161 1522
1523 /*
1524 * If this host crashed, the other host may think the link is
1525 * still up. Tell them to force it down (it will go back up
1526 * once we register the ntb device).
1527 */
1528 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
1529
1162 rc = ntb_register_device(&sndev->ntb); 1530 rc = ntb_register_device(&sndev->ntb);
1163 if (rc) 1531 if (rc)
1164 goto deinit_and_exit; 1532 goto deinit_and_exit;
1165 1533
1166 stdev->sndev = sndev; 1534 stdev->sndev = sndev;
1167 stdev->link_notifier = switchtec_ntb_link_notification; 1535 stdev->link_notifier = switchtec_ntb_link_notification;
1168 dev_info(dev, "NTB device registered"); 1536 dev_info(dev, "NTB device registered\n");
1169 1537
1170 return 0; 1538 return 0;
1171 1539
@@ -1173,14 +1541,16 @@ deinit_and_exit:
1173 switchtec_ntb_deinit_db_msg_irq(sndev); 1541 switchtec_ntb_deinit_db_msg_irq(sndev);
1174deinit_shared_and_exit: 1542deinit_shared_and_exit:
1175 switchtec_ntb_deinit_shared_mw(sndev); 1543 switchtec_ntb_deinit_shared_mw(sndev);
1544deinit_crosslink:
1545 switchtec_ntb_deinit_crosslink(sndev);
1176free_and_exit: 1546free_and_exit:
1177 kfree(sndev); 1547 kfree(sndev);
1178 dev_err(dev, "failed to register ntb device: %d", rc); 1548 dev_err(dev, "failed to register ntb device: %d\n", rc);
1179 return rc; 1549 return rc;
1180} 1550}
1181 1551
1182void switchtec_ntb_remove(struct device *dev, 1552static void switchtec_ntb_remove(struct device *dev,
1183 struct class_interface *class_intf) 1553 struct class_interface *class_intf)
1184{ 1554{
1185 struct switchtec_dev *stdev = to_stdev(dev); 1555 struct switchtec_dev *stdev = to_stdev(dev);
1186 struct switchtec_ntb *sndev = stdev->sndev; 1556 struct switchtec_ntb *sndev = stdev->sndev;
@@ -1193,8 +1563,9 @@ void switchtec_ntb_remove(struct device *dev,
1193 ntb_unregister_device(&sndev->ntb); 1563 ntb_unregister_device(&sndev->ntb);
1194 switchtec_ntb_deinit_db_msg_irq(sndev); 1564 switchtec_ntb_deinit_db_msg_irq(sndev);
1195 switchtec_ntb_deinit_shared_mw(sndev); 1565 switchtec_ntb_deinit_shared_mw(sndev);
1566 switchtec_ntb_deinit_crosslink(sndev);
1196 kfree(sndev); 1567 kfree(sndev);
1197 dev_info(dev, "ntb device unregistered"); 1568 dev_info(dev, "ntb device unregistered\n");
1198} 1569}
1199 1570
1200static struct class_interface switchtec_interface = { 1571static struct class_interface switchtec_interface = {
diff --git a/drivers/ntb/ntb.c b/drivers/ntb/ntb.c
index 03b80d89b980..2581ab724c34 100644
--- a/drivers/ntb/ntb.c
+++ b/drivers/ntb/ntb.c
@@ -63,12 +63,11 @@
63#define DRIVER_NAME "ntb" 63#define DRIVER_NAME "ntb"
64#define DRIVER_DESCRIPTION "PCIe NTB Driver Framework" 64#define DRIVER_DESCRIPTION "PCIe NTB Driver Framework"
65 65
66#define DRIVER_LICENSE "Dual BSD/GPL"
67#define DRIVER_VERSION "1.0" 66#define DRIVER_VERSION "1.0"
68#define DRIVER_RELDATE "24 March 2015" 67#define DRIVER_RELDATE "24 March 2015"
69#define DRIVER_AUTHOR "Allen Hubbe <Allen.Hubbe@emc.com>" 68#define DRIVER_AUTHOR "Allen Hubbe <Allen.Hubbe@emc.com>"
70 69
71MODULE_LICENSE(DRIVER_LICENSE); 70MODULE_LICENSE("Dual BSD/GPL");
72MODULE_VERSION(DRIVER_VERSION); 71MODULE_VERSION(DRIVER_VERSION);
73MODULE_AUTHOR(DRIVER_AUTHOR); 72MODULE_AUTHOR(DRIVER_AUTHOR);
74MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 73MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
@@ -112,7 +111,6 @@ int ntb_register_device(struct ntb_dev *ntb)
112 111
113 init_completion(&ntb->released); 112 init_completion(&ntb->released);
114 113
115 memset(&ntb->dev, 0, sizeof(ntb->dev));
116 ntb->dev.bus = &ntb_bus; 114 ntb->dev.bus = &ntb_bus;
117 ntb->dev.parent = &ntb->pdev->dev; 115 ntb->dev.parent = &ntb->pdev->dev;
118 ntb->dev.release = ntb_dev_release; 116 ntb->dev.release = ntb_dev_release;
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 045e3dd4750e..9878c48826e3 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -1003,6 +1003,9 @@ static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
1003 mw_base = nt->mw_vec[mw_num].phys_addr; 1003 mw_base = nt->mw_vec[mw_num].phys_addr;
1004 mw_size = nt->mw_vec[mw_num].phys_size; 1004 mw_size = nt->mw_vec[mw_num].phys_size;
1005 1005
1006 if (max_mw_size && mw_size > max_mw_size)
1007 mw_size = max_mw_size;
1008
1006 tx_size = (unsigned int)mw_size / num_qps_mw; 1009 tx_size = (unsigned int)mw_size / num_qps_mw;
1007 qp_offset = tx_size * (qp_num / mw_count); 1010 qp_offset = tx_size * (qp_num / mw_count);
1008 1011
diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
index 427112cf101a..2a9d6b0d1f19 100644
--- a/drivers/ntb/test/ntb_perf.c
+++ b/drivers/ntb/test/ntb_perf.c
@@ -5,6 +5,7 @@
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright(c) 2015 Intel Corporation. All rights reserved. 7 * Copyright(c) 2015 Intel Corporation. All rights reserved.
8 * Copyright(c) 2017 T-Platforms. All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
13 * BSD LICENSE 14 * BSD LICENSE
14 * 15 *
15 * Copyright(c) 2015 Intel Corporation. All rights reserved. 16 * Copyright(c) 2015 Intel Corporation. All rights reserved.
17 * Copyright(c) 2017 T-Platforms. All Rights Reserved.
16 * 18 *
17 * Redistribution and use in source and binary forms, with or without 19 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions 20 * modification, are permitted provided that the following conditions
@@ -40,860 +42,1474 @@
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 * 44 *
43 * PCIe NTB Perf Linux driver 45 * PCIe NTB Perf Linux driver
46 */
47
48/*
49 * How to use this tool, by example.
50 *
51 * Assuming $DBG_DIR is something like:
52 * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
53 * Suppose aside from local device there is at least one remote device
54 * connected to NTB with index 0.
55 *-----------------------------------------------------------------------------
56 * Eg: install driver with specified chunk/total orders and dma-enabled flag
57 *
58 * root@self# insmod ntb_perf.ko chunk_order=19 total_order=28 use_dma
59 *-----------------------------------------------------------------------------
60 * Eg: check NTB ports (index) and MW mapping information
61 *
62 * root@self# cat $DBG_DIR/info
63 *-----------------------------------------------------------------------------
64 * Eg: start performance test with peer (index 0) and get the test metrics
65 *
66 * root@self# echo 0 > $DBG_DIR/run
67 * root@self# cat $DBG_DIR/run
44 */ 68 */
45 69
46#include <linux/init.h> 70#include <linux/init.h>
47#include <linux/kernel.h> 71#include <linux/kernel.h>
48#include <linux/module.h> 72#include <linux/module.h>
49#include <linux/kthread.h> 73#include <linux/sched.h>
50#include <linux/time.h> 74#include <linux/wait.h>
51#include <linux/timer.h>
52#include <linux/dma-mapping.h> 75#include <linux/dma-mapping.h>
76#include <linux/dmaengine.h>
53#include <linux/pci.h> 77#include <linux/pci.h>
78#include <linux/ktime.h>
54#include <linux/slab.h> 79#include <linux/slab.h>
55#include <linux/spinlock.h>
56#include <linux/debugfs.h>
57#include <linux/dmaengine.h>
58#include <linux/delay.h> 80#include <linux/delay.h>
59#include <linux/sizes.h> 81#include <linux/sizes.h>
82#include <linux/workqueue.h>
83#include <linux/debugfs.h>
84#include <linux/random.h>
60#include <linux/ntb.h> 85#include <linux/ntb.h>
61#include <linux/mutex.h>
62 86
63#define DRIVER_NAME "ntb_perf" 87#define DRIVER_NAME "ntb_perf"
64#define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool" 88#define DRIVER_VERSION "2.0"
65 89
66#define DRIVER_LICENSE "Dual BSD/GPL" 90MODULE_LICENSE("Dual BSD/GPL");
67#define DRIVER_VERSION "1.0"
68#define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>"
69
70#define PERF_LINK_DOWN_TIMEOUT 10
71#define PERF_VERSION 0xffff0001
72#define MAX_THREADS 32
73#define MAX_TEST_SIZE SZ_1M
74#define MAX_SRCS 32
75#define DMA_OUT_RESOURCE_TO msecs_to_jiffies(50)
76#define DMA_RETRIES 20
77#define SZ_4G (1ULL << 32)
78#define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */
79#define PIDX NTB_DEF_PEER_IDX
80
81MODULE_LICENSE(DRIVER_LICENSE);
82MODULE_VERSION(DRIVER_VERSION); 91MODULE_VERSION(DRIVER_VERSION);
83MODULE_AUTHOR(DRIVER_AUTHOR); 92MODULE_AUTHOR("Dave Jiang <dave.jiang@intel.com>");
84MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 93MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
94
95#define MAX_THREADS_CNT 32
96#define DEF_THREADS_CNT 1
97#define MAX_CHUNK_SIZE SZ_1M
98#define MAX_CHUNK_ORDER 20 /* no larger than 1M */
85 99
86static struct dentry *perf_debugfs_dir; 100#define DMA_TRIES 100
101#define DMA_MDELAY 10
102
103#define MSG_TRIES 500
104#define MSG_UDELAY_LOW 1000
105#define MSG_UDELAY_HIGH 2000
106
107#define PERF_BUF_LEN 1024
87 108
88static unsigned long max_mw_size; 109static unsigned long max_mw_size;
89module_param(max_mw_size, ulong, 0644); 110module_param(max_mw_size, ulong, 0644);
90MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows"); 111MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size");
91 112
92static unsigned int seg_order = 19; /* 512K */ 113static unsigned char chunk_order = 19; /* 512K */
93module_param(seg_order, uint, 0644); 114module_param(chunk_order, byte, 0644);
94MODULE_PARM_DESC(seg_order, "size order [2^n] of buffer segment for testing"); 115MODULE_PARM_DESC(chunk_order, "Data chunk order [2^n] to transfer");
95 116
96static unsigned int run_order = 32; /* 4G */ 117static unsigned char total_order = 30; /* 1G */
97module_param(run_order, uint, 0644); 118module_param(total_order, byte, 0644);
98MODULE_PARM_DESC(run_order, "size order [2^n] of total data to transfer"); 119MODULE_PARM_DESC(total_order, "Total data order [2^n] to transfer");
99 120
100static bool use_dma; /* default to 0 */ 121static bool use_dma; /* default to 0 */
101module_param(use_dma, bool, 0644); 122module_param(use_dma, bool, 0644);
102MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance"); 123MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance");
103 124
104static bool on_node = true; /* default to 1 */ 125/*==============================================================================
105module_param(on_node, bool, 0644); 126 * Perf driver data definition
106MODULE_PARM_DESC(on_node, "Run threads only on NTB device node (default: true)"); 127 *==============================================================================
107 128 */
108struct perf_mw { 129
109 phys_addr_t phys_addr; 130enum perf_cmd {
110 resource_size_t phys_size; 131 PERF_CMD_INVAL = -1,/* invalid spad command */
111 void __iomem *vbase; 132 PERF_CMD_SSIZE = 0, /* send out buffer size */
112 size_t xlat_size; 133 PERF_CMD_RSIZE = 1, /* recv in buffer size */
113 size_t buf_size; 134 PERF_CMD_SXLAT = 2, /* send in buffer xlat */
114 void *virt_addr; 135 PERF_CMD_RXLAT = 3, /* recv out buffer xlat */
115 dma_addr_t dma_addr; 136 PERF_CMD_CLEAR = 4, /* clear allocated memory */
137 PERF_STS_DONE = 5, /* init is done */
138 PERF_STS_LNKUP = 6, /* link up state flag */
116}; 139};
117 140
118struct perf_ctx; 141struct perf_ctx;
119 142
120struct pthr_ctx { 143struct perf_peer {
121 struct task_struct *thread; 144 struct perf_ctx *perf;
122 struct perf_ctx *perf; 145 int pidx;
123 atomic_t dma_sync; 146 int gidx;
124 struct dma_chan *dma_chan; 147
125 int dma_prep_err; 148 /* Outbound MW params */
126 int src_idx; 149 u64 outbuf_xlat;
127 void *srcs[MAX_SRCS]; 150 resource_size_t outbuf_size;
128 wait_queue_head_t *wq; 151 void __iomem *outbuf;
129 int status; 152
130 u64 copied; 153 /* Inbound MW params */
131 u64 diff_us; 154 dma_addr_t inbuf_xlat;
155 resource_size_t inbuf_size;
156 void *inbuf;
157
158 /* NTB connection setup service */
159 struct work_struct service;
160 unsigned long sts;
132}; 161};
162#define to_peer_service(__work) \
163 container_of(__work, struct perf_peer, service)
133 164
134struct perf_ctx { 165struct perf_thread {
135 struct ntb_dev *ntb; 166 struct perf_ctx *perf;
136 spinlock_t db_lock; 167 int tidx;
137 struct perf_mw mw; 168
138 bool link_is_up; 169 /* DMA-based test sync parameters */
139 struct delayed_work link_work; 170 atomic_t dma_sync;
140 wait_queue_head_t link_wq; 171 wait_queue_head_t dma_wait;
141 u8 perf_threads; 172 struct dma_chan *dma_chan;
142 /* mutex ensures only one set of threads run at once */ 173
143 struct mutex run_mutex; 174 /* Data source and measured statistics */
144 struct pthr_ctx pthr_ctx[MAX_THREADS]; 175 void *src;
145 atomic_t tsync; 176 u64 copied;
146 atomic_t tdone; 177 ktime_t duration;
178 int status;
179 struct work_struct work;
147}; 180};
181#define to_thread_work(__work) \
182 container_of(__work, struct perf_thread, work)
148 183
149enum { 184struct perf_ctx {
150 VERSION = 0, 185 struct ntb_dev *ntb;
151 MW_SZ_HIGH, 186
152 MW_SZ_LOW, 187 /* Global device index and peers descriptors */
153 MAX_SPAD 188 int gidx;
189 int pcnt;
190 struct perf_peer *peers;
191
192 /* Performance measuring work-threads interface */
193 unsigned long busy_flag;
194 wait_queue_head_t twait;
195 atomic_t tsync;
196 u8 tcnt;
197 struct perf_peer *test_peer;
198 struct perf_thread threads[MAX_THREADS_CNT];
199
200 /* Scratchpad/Message IO operations */
201 int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
202 int (*cmd_recv)(struct perf_ctx *perf, int *pidx, enum perf_cmd *cmd,
203 u64 *data);
204
205 struct dentry *dbgfs_dir;
154}; 206};
155 207
208/*
209 * Scratchpads-base commands interface
210 */
211#define PERF_SPAD_CNT(_pcnt) \
212 (3*((_pcnt) + 1))
213#define PERF_SPAD_CMD(_gidx) \
214 (3*(_gidx))
215#define PERF_SPAD_LDATA(_gidx) \
216 (3*(_gidx) + 1)
217#define PERF_SPAD_HDATA(_gidx) \
218 (3*(_gidx) + 2)
219#define PERF_SPAD_NOTIFY(_gidx) \
220 (BIT_ULL(_gidx))
221
222/*
223 * Messages-base commands interface
224 */
225#define PERF_MSG_CNT 3
226#define PERF_MSG_CMD 0
227#define PERF_MSG_LDATA 1
228#define PERF_MSG_HDATA 2
229
230/*==============================================================================
231 * Static data declarations
232 *==============================================================================
233 */
234
235static struct dentry *perf_dbgfs_topdir;
236
237static struct workqueue_struct *perf_wq __read_mostly;
238
239/*==============================================================================
240 * NTB cross-link commands execution service
241 *==============================================================================
242 */
243
244static void perf_terminate_test(struct perf_ctx *perf);
245
246static inline bool perf_link_is_up(struct perf_peer *peer)
247{
248 u64 link;
249
250 link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);
251 return !!(link & BIT_ULL_MASK(peer->pidx));
252}
253
254static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
255 u64 data)
256{
257 struct perf_ctx *perf = peer->perf;
258 int try;
259 u32 sts;
260
261 dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
262
263 /*
264 * Perform predefined number of attempts before give up.
265 * We are sending the data to the port specific scratchpad, so
266 * to prevent a multi-port access race-condition. Additionally
267 * there is no need in local locking since only thread-safe
268 * service work is using this method.
269 */
270 for (try = 0; try < MSG_TRIES; try++) {
271 if (!perf_link_is_up(peer))
272 return -ENOLINK;
273
274 sts = ntb_peer_spad_read(perf->ntb, peer->pidx,
275 PERF_SPAD_CMD(perf->gidx));
276 if (sts != PERF_CMD_INVAL) {
277 usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
278 continue;
279 }
280
281 ntb_peer_spad_write(perf->ntb, peer->pidx,
282 PERF_SPAD_LDATA(perf->gidx),
283 lower_32_bits(data));
284 ntb_peer_spad_write(perf->ntb, peer->pidx,
285 PERF_SPAD_HDATA(perf->gidx),
286 upper_32_bits(data));
287 mmiowb();
288 ntb_peer_spad_write(perf->ntb, peer->pidx,
289 PERF_SPAD_CMD(perf->gidx),
290 cmd);
291 mmiowb();
292 ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));
293
294 dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",
295 PERF_SPAD_NOTIFY(peer->gidx));
296
297 break;
298 }
299
300 return try < MSG_TRIES ? 0 : -EAGAIN;
301}
302
303static int perf_spad_cmd_recv(struct perf_ctx *perf, int *pidx,
304 enum perf_cmd *cmd, u64 *data)
305{
306 struct perf_peer *peer;
307 u32 val;
308
309 ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
310
311 /*
312 * We start scanning all over, since cleared DB may have been set
313 * by any peer. Yes, it makes peer with smaller index being
314 * serviced with greater priority, but it's convenient for spad
315 * and message code unification and simplicity.
316 */
317 for (*pidx = 0; *pidx < perf->pcnt; (*pidx)++) {
318 peer = &perf->peers[*pidx];
319
320 if (!perf_link_is_up(peer))
321 continue;
322
323 val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));
324 if (val == PERF_CMD_INVAL)
325 continue;
326
327 *cmd = val;
328
329 val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));
330 *data = val;
331
332 val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));
333 *data |= (u64)val << 32;
334
335 /* Next command can be retrieved from now */
336 ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),
337 PERF_CMD_INVAL);
338
339 dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
340
341 return 0;
342 }
343
344 return -ENODATA;
345}
346
347static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
348 u64 data)
349{
350 struct perf_ctx *perf = peer->perf;
351 int try, ret;
352 u64 outbits;
353
354 dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
355
356 /*
357 * Perform predefined number of attempts before give up. Message
358 * registers are free of race-condition problem when accessed
359 * from different ports, so we don't need splitting registers
360 * by global device index. We also won't have local locking,
361 * since the method is used from service work only.
362 */
363 outbits = ntb_msg_outbits(perf->ntb);
364 for (try = 0; try < MSG_TRIES; try++) {
365 if (!perf_link_is_up(peer))
366 return -ENOLINK;
367
368 ret = ntb_msg_clear_sts(perf->ntb, outbits);
369 if (ret)
370 return ret;
371
372 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,
373 lower_32_bits(data));
374
375 if (ntb_msg_read_sts(perf->ntb) & outbits) {
376 usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
377 continue;
378 }
379
380 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,
381 upper_32_bits(data));
382 mmiowb();
383
384 /* This call shall trigger peer message event */
385 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);
386
387 break;
388 }
389
390 return try < MSG_TRIES ? 0 : -EAGAIN;
391}
392
393static int perf_msg_cmd_recv(struct perf_ctx *perf, int *pidx,
394 enum perf_cmd *cmd, u64 *data)
395{
396 u64 inbits;
397 u32 val;
398
399 inbits = ntb_msg_inbits(perf->ntb);
400
401 if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)
402 return -ENODATA;
403
404 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);
405 *cmd = val;
406
407 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);
408 *data = val;
409
410 val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);
411 *data |= (u64)val << 32;
412
413 /* Next command can be retrieved from now */
414 ntb_msg_clear_sts(perf->ntb, inbits);
415
416 dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
417
418 return 0;
419}
420
421static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)
422{
423 struct perf_ctx *perf = peer->perf;
424
425 if (cmd == PERF_CMD_SSIZE || cmd == PERF_CMD_SXLAT)
426 return perf->cmd_send(peer, cmd, data);
427
428 dev_err(&perf->ntb->dev, "Send invalid command\n");
429 return -EINVAL;
430}
431
432static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)
433{
434 switch (cmd) {
435 case PERF_CMD_SSIZE:
436 case PERF_CMD_RSIZE:
437 case PERF_CMD_SXLAT:
438 case PERF_CMD_RXLAT:
439 case PERF_CMD_CLEAR:
440 break;
441 default:
442 dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");
443 return -EINVAL;
444 }
445
446 /* No need of memory barrier, since bit ops have invernal lock */
447 set_bit(cmd, &peer->sts);
448
449 dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);
450
451 (void)queue_work(system_highpri_wq, &peer->service);
452
453 return 0;
454}
455
456static int perf_cmd_recv(struct perf_ctx *perf)
457{
458 struct perf_peer *peer;
459 int ret, pidx, cmd;
460 u64 data;
461
462 while (!(ret = perf->cmd_recv(perf, &pidx, &cmd, &data))) {
463 peer = &perf->peers[pidx];
464
465 switch (cmd) {
466 case PERF_CMD_SSIZE:
467 peer->inbuf_size = data;
468 return perf_cmd_exec(peer, PERF_CMD_RSIZE);
469 case PERF_CMD_SXLAT:
470 peer->outbuf_xlat = data;
471 return perf_cmd_exec(peer, PERF_CMD_RXLAT);
472 default:
473 dev_err(&perf->ntb->dev, "Recv invalid command\n");
474 return -EINVAL;
475 }
476 }
477
478 /* Return 0 if no data left to process, otherwise an error */
479 return ret == -ENODATA ? 0 : ret;
480}
481
156static void perf_link_event(void *ctx) 482static void perf_link_event(void *ctx)
157{ 483{
158 struct perf_ctx *perf = ctx; 484 struct perf_ctx *perf = ctx;
485 struct perf_peer *peer;
486 bool lnk_up;
487 int pidx;
159 488
160 if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1) { 489 for (pidx = 0; pidx < perf->pcnt; pidx++) {
161 schedule_delayed_work(&perf->link_work, 2*HZ); 490 peer = &perf->peers[pidx];
162 } else {
163 dev_dbg(&perf->ntb->pdev->dev, "link down\n");
164 491
165 if (!perf->link_is_up) 492 lnk_up = perf_link_is_up(peer);
166 cancel_delayed_work_sync(&perf->link_work);
167 493
168 perf->link_is_up = false; 494 if (lnk_up &&
495 !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {
496 perf_cmd_exec(peer, PERF_CMD_SSIZE);
497 } else if (!lnk_up &&
498 test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {
499 perf_cmd_exec(peer, PERF_CMD_CLEAR);
500 }
169 } 501 }
170} 502}
171 503
172static void perf_db_event(void *ctx, int vec) 504static void perf_db_event(void *ctx, int vec)
173{ 505{
174 struct perf_ctx *perf = ctx; 506 struct perf_ctx *perf = ctx;
175 u64 db_bits, db_mask;
176 507
177 db_mask = ntb_db_vector_mask(perf->ntb, vec); 508 dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,
178 db_bits = ntb_db_read(perf->ntb); 509 ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));
510
511 /* Just receive all available commands */
512 (void)perf_cmd_recv(perf);
513}
514
515static void perf_msg_event(void *ctx)
516{
517 struct perf_ctx *perf = ctx;
518
519 dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",
520 ntb_msg_read_sts(perf->ntb));
179 521
180 dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n", 522 /* Messages are only sent one-by-one */
181 vec, db_mask, db_bits); 523 (void)perf_cmd_recv(perf);
182} 524}
183 525
184static const struct ntb_ctx_ops perf_ops = { 526static const struct ntb_ctx_ops perf_ops = {
185 .link_event = perf_link_event, 527 .link_event = perf_link_event,
186 .db_event = perf_db_event, 528 .db_event = perf_db_event,
529 .msg_event = perf_msg_event
187}; 530};
188 531
189static void perf_copy_callback(void *data) 532static void perf_free_outbuf(struct perf_peer *peer)
533{
534 (void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
535}
536
537static int perf_setup_outbuf(struct perf_peer *peer)
190{ 538{
191 struct pthr_ctx *pctx = data; 539 struct perf_ctx *perf = peer->perf;
540 int ret;
541
542 /* Outbuf size can be unaligned due to custom max_mw_size */
543 ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
544 peer->outbuf_xlat, peer->outbuf_size);
545 if (ret) {
546 dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");
547 return ret;
548 }
192 549
193 atomic_dec(&pctx->dma_sync); 550 /* Initialization is finally done */
551 set_bit(PERF_STS_DONE, &peer->sts);
552
553 return 0;
194} 554}
195 555
196static ssize_t perf_copy(struct pthr_ctx *pctx, char __iomem *dst, 556static void perf_free_inbuf(struct perf_peer *peer)
197 char *src, size_t size)
198{ 557{
199 struct perf_ctx *perf = pctx->perf; 558 if (!peer->inbuf)
200 struct dma_async_tx_descriptor *txd; 559 return;
201 struct dma_chan *chan = pctx->dma_chan;
202 struct dma_device *device;
203 struct dmaengine_unmap_data *unmap;
204 dma_cookie_t cookie;
205 size_t src_off, dst_off;
206 struct perf_mw *mw = &perf->mw;
207 void __iomem *vbase;
208 void __iomem *dst_vaddr;
209 dma_addr_t dst_phys;
210 int retries = 0;
211 560
212 if (!use_dma) { 561 (void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
213 memcpy_toio(dst, src, size); 562 dma_free_coherent(&peer->perf->ntb->dev, peer->inbuf_size,
214 return size; 563 peer->inbuf, peer->inbuf_xlat);
564 peer->inbuf = NULL;
565}
566
567static int perf_setup_inbuf(struct perf_peer *peer)
568{
569 resource_size_t xlat_align, size_align, size_max;
570 struct perf_ctx *perf = peer->perf;
571 int ret;
572
573 /* Get inbound MW parameters */
574 ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,
575 &xlat_align, &size_align, &size_max);
576 if (ret) {
577 dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");
578 return ret;
215 } 579 }
216 580
217 if (!chan) { 581 if (peer->inbuf_size > size_max) {
218 dev_err(&perf->ntb->dev, "DMA engine does not exist\n"); 582 dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",
583 &peer->inbuf_size, &size_max);
219 return -EINVAL; 584 return -EINVAL;
220 } 585 }
221 586
222 device = chan->device; 587 peer->inbuf_size = round_up(peer->inbuf_size, size_align);
223 src_off = (uintptr_t)src & ~PAGE_MASK;
224 dst_off = (uintptr_t __force)dst & ~PAGE_MASK;
225
226 if (!is_dma_copy_aligned(device, src_off, dst_off, size))
227 return -ENODEV;
228 588
229 vbase = mw->vbase; 589 perf_free_inbuf(peer);
230 dst_vaddr = dst;
231 dst_phys = mw->phys_addr + (dst_vaddr - vbase);
232 590
233 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT); 591 peer->inbuf = dma_alloc_coherent(&perf->ntb->dev, peer->inbuf_size,
234 if (!unmap) 592 &peer->inbuf_xlat, GFP_KERNEL);
593 if (!peer->inbuf) {
594 dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
595 &peer->inbuf_size);
235 return -ENOMEM; 596 return -ENOMEM;
597 }
598 if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {
599 dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");
600 goto err_free_inbuf;
601 }
236 602
237 unmap->len = size; 603 ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
238 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src), 604 peer->inbuf_xlat, peer->inbuf_size);
239 src_off, size, DMA_TO_DEVICE); 605 if (ret) {
240 if (dma_mapping_error(device->dev, unmap->addr[0])) 606 dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");
241 goto err_get_unmap; 607 goto err_free_inbuf;
608 }
242 609
243 unmap->to_cnt = 1; 610 /*
611 * We submit inbuf xlat transmission cmd for execution here to follow
612 * the code architecture, even though this method is called from service
613 * work itself so the command will be executed right after it returns.
614 */
615 (void)perf_cmd_exec(peer, PERF_CMD_SXLAT);
244 616
245 do { 617 return 0;
246 txd = device->device_prep_dma_memcpy(chan, dst_phys,
247 unmap->addr[0],
248 size, DMA_PREP_INTERRUPT);
249 if (!txd) {
250 set_current_state(TASK_INTERRUPTIBLE);
251 schedule_timeout(DMA_OUT_RESOURCE_TO);
252 }
253 } while (!txd && (++retries < DMA_RETRIES));
254 618
255 if (!txd) { 619err_free_inbuf:
256 pctx->dma_prep_err++; 620 perf_free_inbuf(peer);
257 goto err_get_unmap;
258 }
259 621
260 txd->callback = perf_copy_callback; 622 return ret;
261 txd->callback_param = pctx; 623}
262 dma_set_unmap(txd, unmap);
263 624
264 cookie = dmaengine_submit(txd); 625static void perf_service_work(struct work_struct *work)
265 if (dma_submit_error(cookie)) 626{
266 goto err_set_unmap; 627 struct perf_peer *peer = to_peer_service(work);
267 628
268 dmaengine_unmap_put(unmap); 629 if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))
630 perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);
269 631
270 atomic_inc(&pctx->dma_sync); 632 if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))
271 dma_async_issue_pending(chan); 633 perf_setup_inbuf(peer);
272 634
273 return size; 635 if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))
636 perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);
274 637
275err_set_unmap: 638 if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))
276 dmaengine_unmap_put(unmap); 639 perf_setup_outbuf(peer);
277err_get_unmap:
278 dmaengine_unmap_put(unmap);
279 return 0;
280}
281 640
282static int perf_move_data(struct pthr_ctx *pctx, char __iomem *dst, char *src, 641 if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {
283 u64 buf_size, u64 win_size, u64 total) 642 clear_bit(PERF_STS_DONE, &peer->sts);
284{ 643 if (test_bit(0, &peer->perf->busy_flag) &&
285 int chunks, total_chunks, i; 644 peer == peer->perf->test_peer) {
286 int copied_chunks = 0; 645 dev_warn(&peer->perf->ntb->dev,
287 u64 copied = 0, result; 646 "Freeing while test on-fly\n");
288 char __iomem *tmp = dst; 647 perf_terminate_test(peer->perf);
289 u64 perf, diff_us;
290 ktime_t kstart, kstop, kdiff;
291 unsigned long last_sleep = jiffies;
292
293 chunks = div64_u64(win_size, buf_size);
294 total_chunks = div64_u64(total, buf_size);
295 kstart = ktime_get();
296
297 for (i = 0; i < total_chunks; i++) {
298 result = perf_copy(pctx, tmp, src, buf_size);
299 copied += result;
300 copied_chunks++;
301 if (copied_chunks == chunks) {
302 tmp = dst;
303 copied_chunks = 0;
304 } else
305 tmp += buf_size;
306
307 /* Probably should schedule every 5s to prevent soft hang. */
308 if (unlikely((jiffies - last_sleep) > 5 * HZ)) {
309 last_sleep = jiffies;
310 set_current_state(TASK_INTERRUPTIBLE);
311 schedule_timeout(1);
312 } 648 }
649 perf_free_outbuf(peer);
650 perf_free_inbuf(peer);
651 }
652}
653
654static int perf_init_service(struct perf_ctx *perf)
655{
656 u64 mask;
313 657
314 if (unlikely(kthread_should_stop())) 658 if (ntb_peer_mw_count(perf->ntb) < perf->pcnt + 1) {
315 break; 659 dev_err(&perf->ntb->dev, "Not enough memory windows\n");
660 return -EINVAL;
316 } 661 }
317 662
318 if (use_dma) { 663 if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {
319 pr_debug("%s: All DMA descriptors submitted\n", current->comm); 664 perf->cmd_send = perf_msg_cmd_send;
320 while (atomic_read(&pctx->dma_sync) != 0) { 665 perf->cmd_recv = perf_msg_cmd_recv;
321 if (kthread_should_stop()) 666
322 break; 667 dev_dbg(&perf->ntb->dev, "Message service initialized\n");
323 msleep(20); 668
324 } 669 return 0;
325 } 670 }
326 671
327 kstop = ktime_get(); 672 dev_dbg(&perf->ntb->dev, "Message service unsupported\n");
328 kdiff = ktime_sub(kstop, kstart);
329 diff_us = ktime_to_us(kdiff);
330 673
331 pr_debug("%s: copied %llu bytes\n", current->comm, copied); 674 mask = GENMASK_ULL(perf->pcnt, 0);
675 if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&
676 (ntb_db_valid_mask(perf->ntb) & mask) == mask) {
677 perf->cmd_send = perf_spad_cmd_send;
678 perf->cmd_recv = perf_spad_cmd_recv;
332 679
333 pr_debug("%s: lasted %llu usecs\n", current->comm, diff_us); 680 dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");
334 681
335 perf = div64_u64(copied, diff_us); 682 return 0;
683 }
336 684
337 pr_debug("%s: MBytes/s: %llu\n", current->comm, perf); 685 dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");
338 686
339 pctx->copied = copied; 687 dev_err(&perf->ntb->dev, "Command services unsupported\n");
340 pctx->diff_us = diff_us;
341 688
342 return 0; 689 return -EINVAL;
343} 690}
344 691
345static bool perf_dma_filter_fn(struct dma_chan *chan, void *node) 692static int perf_enable_service(struct perf_ctx *perf)
346{ 693{
347 /* Is the channel required to be on the same node as the device? */ 694 u64 mask, incmd_bit;
348 if (!on_node) 695 int ret, sidx, scnt;
349 return true;
350 696
351 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node; 697 mask = ntb_db_valid_mask(perf->ntb);
352} 698 (void)ntb_db_set_mask(perf->ntb, mask);
353 699
354static int ntb_perf_thread(void *data) 700 ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);
355{ 701 if (ret)
356 struct pthr_ctx *pctx = data; 702 return ret;
357 struct perf_ctx *perf = pctx->perf;
358 struct pci_dev *pdev = perf->ntb->pdev;
359 struct perf_mw *mw = &perf->mw;
360 char __iomem *dst;
361 u64 win_size, buf_size, total;
362 void *src;
363 int rc, node, i;
364 struct dma_chan *dma_chan = NULL;
365 703
366 pr_debug("kthread %s starting...\n", current->comm); 704 if (perf->cmd_send == perf_msg_cmd_send) {
705 u64 inbits, outbits;
367 706
368 node = on_node ? dev_to_node(&pdev->dev) : NUMA_NO_NODE; 707 inbits = ntb_msg_inbits(perf->ntb);
708 outbits = ntb_msg_outbits(perf->ntb);
709 (void)ntb_msg_set_mask(perf->ntb, inbits | outbits);
369 710
370 if (use_dma && !pctx->dma_chan) { 711 incmd_bit = BIT_ULL(__ffs64(inbits));
371 dma_cap_mask_t dma_mask; 712 ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);
372 713
373 dma_cap_zero(dma_mask); 714 dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);
374 dma_cap_set(DMA_MEMCPY, dma_mask); 715 } else {
375 dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn, 716 scnt = ntb_spad_count(perf->ntb);
376 (void *)(unsigned long)node); 717 for (sidx = 0; sidx < scnt; sidx++)
377 if (!dma_chan) { 718 ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);
378 pr_warn("%s: cannot acquire DMA channel, quitting\n", 719 incmd_bit = PERF_SPAD_NOTIFY(perf->gidx);
379 current->comm); 720 ret = ntb_db_clear_mask(perf->ntb, incmd_bit);
380 return -ENODEV; 721
381 } 722 dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);
382 pctx->dma_chan = dma_chan; 723 }
724 if (ret) {
725 ntb_clear_ctx(perf->ntb);
726 return ret;
383 } 727 }
384 728
385 for (i = 0; i < MAX_SRCS; i++) { 729 ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
386 pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node); 730 /* Might be not necessary */
387 if (!pctx->srcs[i]) { 731 ntb_link_event(perf->ntb);
388 rc = -ENOMEM; 732
389 goto err; 733 return 0;
390 } 734}
735
736static void perf_disable_service(struct perf_ctx *perf)
737{
738 int pidx;
739
740 ntb_link_disable(perf->ntb);
741
742 if (perf->cmd_send == perf_msg_cmd_send) {
743 u64 inbits;
744
745 inbits = ntb_msg_inbits(perf->ntb);
746 (void)ntb_msg_set_mask(perf->ntb, inbits);
747 } else {
748 (void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
391 } 749 }
392 750
393 win_size = mw->phys_size; 751 ntb_clear_ctx(perf->ntb);
394 buf_size = 1ULL << seg_order;
395 total = 1ULL << run_order;
396 752
397 if (buf_size > MAX_TEST_SIZE) 753 for (pidx = 0; pidx < perf->pcnt; pidx++)
398 buf_size = MAX_TEST_SIZE; 754 perf_cmd_exec(&perf->peers[pidx], PERF_CMD_CLEAR);
399 755
400 dst = (char __iomem *)mw->vbase; 756 for (pidx = 0; pidx < perf->pcnt; pidx++)
757 flush_work(&perf->peers[pidx].service);
758}
401 759
402 atomic_inc(&perf->tsync); 760/*==============================================================================
403 while (atomic_read(&perf->tsync) != perf->perf_threads) 761 * Performance measuring work-thread
404 schedule(); 762 *==============================================================================
763 */
405 764
406 src = pctx->srcs[pctx->src_idx]; 765static void perf_dma_copy_callback(void *data)
407 pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1); 766{
767 struct perf_thread *pthr = data;
408 768
409 rc = perf_move_data(pctx, dst, src, buf_size, win_size, total); 769 atomic_dec(&pthr->dma_sync);
770 wake_up(&pthr->dma_wait);
771}
410 772
411 atomic_dec(&perf->tsync); 773static int perf_copy_chunk(struct perf_thread *pthr,
774 void __iomem *dst, void *src, size_t len)
775{
776 struct dma_async_tx_descriptor *tx;
777 struct dmaengine_unmap_data *unmap;
778 struct device *dma_dev;
779 int try = 0, ret = 0;
412 780
413 if (rc < 0) { 781 if (!use_dma) {
414 pr_err("%s: failed\n", current->comm); 782 memcpy_toio(dst, src, len);
415 rc = -ENXIO; 783 goto ret_check_tsync;
416 goto err;
417 } 784 }
418 785
419 for (i = 0; i < MAX_SRCS; i++) { 786 dma_dev = pthr->dma_chan->device->dev;
420 kfree(pctx->srcs[i]); 787
421 pctx->srcs[i] = NULL; 788 if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),
789 offset_in_page(dst), len))
790 return -EIO;
791
792 unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
793 if (!unmap)
794 return -ENOMEM;
795
796 unmap->len = len;
797 unmap->addr[0] = dma_map_page(dma_dev, virt_to_page(src),
798 offset_in_page(src), len, DMA_TO_DEVICE);
799 if (dma_mapping_error(dma_dev, unmap->addr[0])) {
800 ret = -EIO;
801 goto err_free_resource;
422 } 802 }
803 unmap->to_cnt = 1;
423 804
424 atomic_inc(&perf->tdone); 805 unmap->addr[1] = dma_map_page(dma_dev, virt_to_page(dst),
425 wake_up(pctx->wq); 806 offset_in_page(dst), len, DMA_FROM_DEVICE);
426 rc = 0; 807 if (dma_mapping_error(dma_dev, unmap->addr[1])) {
427 goto done; 808 ret = -EIO;
809 goto err_free_resource;
810 }
811 unmap->from_cnt = 1;
428 812
429err: 813 do {
430 for (i = 0; i < MAX_SRCS; i++) { 814 tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, unmap->addr[1],
431 kfree(pctx->srcs[i]); 815 unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
432 pctx->srcs[i] = NULL; 816 if (!tx)
817 msleep(DMA_MDELAY);
818 } while (!tx && (try++ < DMA_TRIES));
819
820 if (!tx) {
821 ret = -EIO;
822 goto err_free_resource;
433 } 823 }
434 824
435 if (dma_chan) { 825 tx->callback = perf_dma_copy_callback;
436 dma_release_channel(dma_chan); 826 tx->callback_param = pthr;
437 pctx->dma_chan = NULL; 827 dma_set_unmap(tx, unmap);
828
829 ret = dma_submit_error(dmaengine_submit(tx));
830 if (ret) {
831 dmaengine_unmap_put(unmap);
832 goto err_free_resource;
438 } 833 }
439 834
440done: 835 dmaengine_unmap_put(unmap);
441 /* Wait until we are told to stop */ 836
442 for (;;) { 837 atomic_inc(&pthr->dma_sync);
443 set_current_state(TASK_INTERRUPTIBLE); 838 dma_async_issue_pending(pthr->dma_chan);
444 if (kthread_should_stop()) 839
445 break; 840ret_check_tsync:
446 schedule(); 841 return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
842
843err_free_resource:
844 dmaengine_unmap_put(unmap);
845
846 return ret;
847}
848
849static bool perf_dma_filter(struct dma_chan *chan, void *data)
850{
851 struct perf_ctx *perf = data;
852 int node;
853
854 node = dev_to_node(&perf->ntb->dev);
855
856 return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
857}
858
859static int perf_init_test(struct perf_thread *pthr)
860{
861 struct perf_ctx *perf = pthr->perf;
862 dma_cap_mask_t dma_mask;
863
864 pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
865 dev_to_node(&perf->ntb->dev));
866 if (!pthr->src)
867 return -ENOMEM;
868
869 get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
870
871 if (!use_dma)
872 return 0;
873
874 dma_cap_zero(dma_mask);
875 dma_cap_set(DMA_MEMCPY, dma_mask);
876 pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
877 if (!pthr->dma_chan) {
878 dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
879 pthr->tidx);
880 atomic_dec(&perf->tsync);
881 wake_up(&perf->twait);
882 kfree(pthr->src);
883 return -ENODEV;
447 } 884 }
448 __set_current_state(TASK_RUNNING);
449 885
450 return rc; 886 atomic_set(&pthr->dma_sync, 0);
887
888 return 0;
451} 889}
452 890
453static void perf_free_mw(struct perf_ctx *perf) 891static int perf_run_test(struct perf_thread *pthr)
454{ 892{
455 struct perf_mw *mw = &perf->mw; 893 struct perf_peer *peer = pthr->perf->test_peer;
456 struct pci_dev *pdev = perf->ntb->pdev; 894 struct perf_ctx *perf = pthr->perf;
895 void __iomem *flt_dst, *bnd_dst;
896 u64 total_size, chunk_size;
897 void *flt_src;
898 int ret = 0;
899
900 total_size = 1ULL << total_order;
901 chunk_size = 1ULL << chunk_order;
902 chunk_size = min_t(u64, peer->outbuf_size, chunk_size);
903
904 flt_src = pthr->src;
905 bnd_dst = peer->outbuf + peer->outbuf_size;
906 flt_dst = peer->outbuf;
907
908 pthr->duration = ktime_get();
909
910 /* Copied field is cleared on test launch stage */
911 while (pthr->copied < total_size) {
912 ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size);
913 if (ret) {
914 dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",
915 pthr->tidx, ret);
916 return ret;
917 }
457 918
458 if (!mw->virt_addr) 919 pthr->copied += chunk_size;
459 return; 920
921 flt_dst += chunk_size;
922 flt_src += chunk_size;
923 if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {
924 flt_dst = peer->outbuf;
925 flt_src = pthr->src;
926 }
460 927
461 ntb_mw_clear_trans(perf->ntb, PIDX, 0); 928 /* Give up CPU to give a chance for other threads to use it */
462 dma_free_coherent(&pdev->dev, mw->buf_size, 929 schedule();
463 mw->virt_addr, mw->dma_addr); 930 }
464 mw->xlat_size = 0; 931
465 mw->buf_size = 0; 932 return 0;
466 mw->virt_addr = NULL;
467} 933}
468 934
469static int perf_set_mw(struct perf_ctx *perf, resource_size_t size) 935static int perf_sync_test(struct perf_thread *pthr)
470{ 936{
471 struct perf_mw *mw = &perf->mw; 937 struct perf_ctx *perf = pthr->perf;
472 size_t xlat_size, buf_size;
473 resource_size_t xlat_align;
474 resource_size_t xlat_align_size;
475 int rc;
476 938
477 if (!size) 939 if (!use_dma)
478 return -EINVAL; 940 goto no_dma_ret;
479 941
480 rc = ntb_mw_get_align(perf->ntb, PIDX, 0, &xlat_align, 942 wait_event(pthr->dma_wait,
481 &xlat_align_size, NULL); 943 (atomic_read(&pthr->dma_sync) == 0 ||
482 if (rc) 944 atomic_read(&perf->tsync) < 0));
483 return rc;
484 945
485 xlat_size = round_up(size, xlat_align_size); 946 if (atomic_read(&perf->tsync) < 0)
486 buf_size = round_up(size, xlat_align); 947 return -EINTR;
487 948
488 if (mw->xlat_size == xlat_size) 949no_dma_ret:
489 return 0; 950 pthr->duration = ktime_sub(ktime_get(), pthr->duration);
490 951
491 if (mw->buf_size) 952 dev_dbg(&perf->ntb->dev, "%d: copied %llu bytes\n",
492 perf_free_mw(perf); 953 pthr->tidx, pthr->copied);
493 954
494 mw->xlat_size = xlat_size; 955 dev_dbg(&perf->ntb->dev, "%d: lasted %llu usecs\n",
495 mw->buf_size = buf_size; 956 pthr->tidx, ktime_to_us(pthr->duration));
957
958 dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,
959 div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
960
961 return 0;
962}
963
964static void perf_clear_test(struct perf_thread *pthr)
965{
966 struct perf_ctx *perf = pthr->perf;
967
968 if (!use_dma)
969 goto no_dma_notify;
970
971 /*
972 * If test finished without errors, termination isn't needed.
973 * We call it anyway just to be sure of the transfers completion.
974 */
975 (void)dmaengine_terminate_sync(pthr->dma_chan);
976
977 dma_release_channel(pthr->dma_chan);
978
979no_dma_notify:
980 atomic_dec(&perf->tsync);
981 wake_up(&perf->twait);
982 kfree(pthr->src);
983}
496 984
497 mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size, 985static void perf_thread_work(struct work_struct *work)
498 &mw->dma_addr, GFP_KERNEL); 986{
499 if (!mw->virt_addr) { 987 struct perf_thread *pthr = to_thread_work(work);
500 mw->xlat_size = 0; 988 int ret;
501 mw->buf_size = 0; 989
990 /*
991 * Perform stages in compliance with use_dma flag value.
992 * Test status is changed only if error happened, otherwise
993 * status -ENODATA is kept while test is on-fly. Results
994 * synchronization is performed only if test fininshed
995 * without an error or interruption.
996 */
997 ret = perf_init_test(pthr);
998 if (ret) {
999 pthr->status = ret;
1000 return;
502 } 1001 }
503 1002
504 rc = ntb_mw_set_trans(perf->ntb, PIDX, 0, mw->dma_addr, mw->xlat_size); 1003 ret = perf_run_test(pthr);
505 if (rc) { 1004 if (ret) {
506 dev_err(&perf->ntb->dev, "Unable to set mw0 translation\n"); 1005 pthr->status = ret;
507 perf_free_mw(perf); 1006 goto err_clear_test;
508 return -EIO;
509 } 1007 }
510 1008
511 return 0; 1009 pthr->status = perf_sync_test(pthr);
1010
1011err_clear_test:
1012 perf_clear_test(pthr);
512} 1013}
513 1014
514static void perf_link_work(struct work_struct *work) 1015static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
515{ 1016{
516 struct perf_ctx *perf = 1017 if (tcnt == 0 || tcnt > MAX_THREADS_CNT)
517 container_of(work, struct perf_ctx, link_work.work); 1018 return -EINVAL;
518 struct ntb_dev *ndev = perf->ntb;
519 struct pci_dev *pdev = ndev->pdev;
520 u32 val;
521 u64 size;
522 int rc;
523 1019
524 dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__); 1020 if (test_and_set_bit_lock(0, &perf->busy_flag))
1021 return -EBUSY;
1022
1023 perf->tcnt = tcnt;
1024
1025 clear_bit_unlock(0, &perf->busy_flag);
525 1026
526 size = perf->mw.phys_size; 1027 return 0;
1028}
527 1029
528 if (max_mw_size && size > max_mw_size) 1030static void perf_terminate_test(struct perf_ctx *perf)
529 size = max_mw_size; 1031{
1032 int tidx;
530 1033
531 ntb_peer_spad_write(ndev, PIDX, MW_SZ_HIGH, upper_32_bits(size)); 1034 atomic_set(&perf->tsync, -1);
532 ntb_peer_spad_write(ndev, PIDX, MW_SZ_LOW, lower_32_bits(size)); 1035 wake_up(&perf->twait);
533 ntb_peer_spad_write(ndev, PIDX, VERSION, PERF_VERSION);
534 1036
535 /* now read what peer wrote */ 1037 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
536 val = ntb_spad_read(ndev, VERSION); 1038 wake_up(&perf->threads[tidx].dma_wait);
537 if (val != PERF_VERSION) { 1039 cancel_work_sync(&perf->threads[tidx].work);
538 dev_dbg(&pdev->dev, "Remote version = %#x\n", val);
539 goto out;
540 } 1040 }
1041}
1042
1043static int perf_submit_test(struct perf_peer *peer)
1044{
1045 struct perf_ctx *perf = peer->perf;
1046 struct perf_thread *pthr;
1047 int tidx, ret;
541 1048
542 val = ntb_spad_read(ndev, MW_SZ_HIGH); 1049 if (!test_bit(PERF_STS_DONE, &peer->sts))
543 size = (u64)val << 32; 1050 return -ENOLINK;
544 1051
545 val = ntb_spad_read(ndev, MW_SZ_LOW); 1052 if (test_and_set_bit_lock(0, &perf->busy_flag))
546 size |= val; 1053 return -EBUSY;
547 1054
548 dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size); 1055 perf->test_peer = peer;
1056 atomic_set(&perf->tsync, perf->tcnt);
549 1057
550 rc = perf_set_mw(perf, size); 1058 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
551 if (rc) 1059 pthr = &perf->threads[tidx];
552 goto out1;
553 1060
554 perf->link_is_up = true; 1061 pthr->status = -ENODATA;
555 wake_up(&perf->link_wq); 1062 pthr->copied = 0;
1063 pthr->duration = ktime_set(0, 0);
1064 if (tidx < perf->tcnt)
1065 (void)queue_work(perf_wq, &pthr->work);
1066 }
556 1067
557 return; 1068 ret = wait_event_interruptible(perf->twait,
1069 atomic_read(&perf->tsync) <= 0);
1070 if (ret == -ERESTARTSYS) {
1071 perf_terminate_test(perf);
1072 ret = -EINTR;
1073 }
558 1074
559out1: 1075 clear_bit_unlock(0, &perf->busy_flag);
560 perf_free_mw(perf);
561 1076
562out: 1077 return ret;
563 if (ntb_link_is_up(ndev, NULL, NULL) == 1)
564 schedule_delayed_work(&perf->link_work,
565 msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT));
566} 1078}
567 1079
568static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf) 1080static int perf_read_stats(struct perf_ctx *perf, char *buf,
1081 size_t size, ssize_t *pos)
569{ 1082{
570 struct perf_mw *mw; 1083 struct perf_thread *pthr;
571 int rc; 1084 int tidx;
1085
1086 if (test_and_set_bit_lock(0, &perf->busy_flag))
1087 return -EBUSY;
572 1088
573 mw = &perf->mw; 1089 (*pos) += scnprintf(buf + *pos, size - *pos,
1090 " Peer %d test statistics:\n", perf->test_peer->pidx);
574 1091
575 rc = ntb_peer_mw_get_addr(ntb, 0, &mw->phys_addr, &mw->phys_size); 1092 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
576 if (rc) 1093 pthr = &perf->threads[tidx];
577 return rc;
578 1094
579 perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size); 1095 if (pthr->status == -ENODATA)
580 if (!mw->vbase) 1096 continue;
581 return -ENOMEM; 1097
1098 if (pthr->status) {
1099 (*pos) += scnprintf(buf + *pos, size - *pos,
1100 "%d: error status %d\n", tidx, pthr->status);
1101 continue;
1102 }
1103
1104 (*pos) += scnprintf(buf + *pos, size - *pos,
1105 "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
1106 tidx, pthr->copied, ktime_to_us(pthr->duration),
1107 div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
1108 }
1109
1110 clear_bit_unlock(0, &perf->busy_flag);
582 1111
583 return 0; 1112 return 0;
584} 1113}
585 1114
586static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf, 1115static void perf_init_threads(struct perf_ctx *perf)
587 size_t count, loff_t *offp)
588{ 1116{
589 struct perf_ctx *perf = filp->private_data; 1117 struct perf_thread *pthr;
1118 int tidx;
1119
1120 perf->tcnt = DEF_THREADS_CNT;
1121 perf->test_peer = &perf->peers[0];
1122 init_waitqueue_head(&perf->twait);
1123
1124 for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1125 pthr = &perf->threads[tidx];
1126
1127 pthr->perf = perf;
1128 pthr->tidx = tidx;
1129 pthr->status = -ENODATA;
1130 init_waitqueue_head(&pthr->dma_wait);
1131 INIT_WORK(&pthr->work, perf_thread_work);
1132 }
1133}
1134
1135static void perf_clear_threads(struct perf_ctx *perf)
1136{
1137 perf_terminate_test(perf);
1138}
1139
1140/*==============================================================================
1141 * DebugFS nodes
1142 *==============================================================================
1143 */
1144
1145static ssize_t perf_dbgfs_read_info(struct file *filep, char __user *ubuf,
1146 size_t size, loff_t *offp)
1147{
1148 struct perf_ctx *perf = filep->private_data;
1149 struct perf_peer *peer;
1150 size_t buf_size;
1151 ssize_t pos = 0;
1152 int ret, pidx;
590 char *buf; 1153 char *buf;
591 ssize_t ret, out_off = 0;
592 struct pthr_ctx *pctx;
593 int i;
594 u64 rate;
595 1154
596 if (!perf) 1155 buf_size = min_t(size_t, size, 0x1000U);
597 return 0;
598 1156
599 buf = kmalloc(1024, GFP_KERNEL); 1157 buf = kmalloc(buf_size, GFP_KERNEL);
600 if (!buf) 1158 if (!buf)
601 return -ENOMEM; 1159 return -ENOMEM;
602 1160
603 if (mutex_is_locked(&perf->run_mutex)) { 1161 pos += scnprintf(buf + pos, buf_size - pos,
604 out_off = scnprintf(buf, 64, "running\n"); 1162 " Performance measuring tool info:\n\n");
605 goto read_from_buf; 1163
1164 pos += scnprintf(buf + pos, buf_size - pos,
1165 "Local port %d, Global index %d\n", ntb_port_number(perf->ntb),
1166 perf->gidx);
1167 pos += scnprintf(buf + pos, buf_size - pos, "Test status: ");
1168 if (test_bit(0, &perf->busy_flag)) {
1169 pos += scnprintf(buf + pos, buf_size - pos,
1170 "on-fly with port %d (%d)\n",
1171 ntb_peer_port_number(perf->ntb, perf->test_peer->pidx),
1172 perf->test_peer->pidx);
1173 } else {
1174 pos += scnprintf(buf + pos, buf_size - pos, "idle\n");
606 } 1175 }
607 1176
608 for (i = 0; i < MAX_THREADS; i++) { 1177 for (pidx = 0; pidx < perf->pcnt; pidx++) {
609 pctx = &perf->pthr_ctx[i]; 1178 peer = &perf->peers[pidx];
1179
1180 pos += scnprintf(buf + pos, buf_size - pos,
1181 "Port %d (%d), Global index %d:\n",
1182 ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,
1183 peer->gidx);
1184
1185 pos += scnprintf(buf + pos, buf_size - pos,
1186 "\tLink status: %s\n",
1187 test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");
1188
1189 pos += scnprintf(buf + pos, buf_size - pos,
1190 "\tOut buffer addr 0x%pK\n", peer->outbuf);
610 1191
611 if (pctx->status == -ENODATA) 1192 pos += scnprintf(buf + pos, buf_size - pos,
612 break; 1193 "\tOut buffer size %pa\n", &peer->outbuf_size);
613 1194
614 if (pctx->status) { 1195 pos += scnprintf(buf + pos, buf_size - pos,
615 out_off += scnprintf(buf + out_off, 1024 - out_off, 1196 "\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);
616 "%d: error %d\n", i, 1197
617 pctx->status); 1198 if (!peer->inbuf) {
1199 pos += scnprintf(buf + pos, buf_size - pos,
1200 "\tIn buffer addr: unallocated\n");
618 continue; 1201 continue;
619 } 1202 }
620 1203
621 rate = div64_u64(pctx->copied, pctx->diff_us); 1204 pos += scnprintf(buf + pos, buf_size - pos,
622 out_off += scnprintf(buf + out_off, 1024 - out_off, 1205 "\tIn buffer addr 0x%pK\n", peer->inbuf);
623 "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n", 1206
624 i, pctx->copied, pctx->diff_us, rate); 1207 pos += scnprintf(buf + pos, buf_size - pos,
1208 "\tIn buffer size %pa\n", &peer->inbuf_size);
1209
1210 pos += scnprintf(buf + pos, buf_size - pos,
1211 "\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);
625 } 1212 }
626 1213
627read_from_buf: 1214 ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
628 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_off);
629 kfree(buf); 1215 kfree(buf);
630 1216
631 return ret; 1217 return ret;
632} 1218}
633 1219
634static void threads_cleanup(struct perf_ctx *perf) 1220static const struct file_operations perf_dbgfs_info = {
1221 .open = simple_open,
1222 .read = perf_dbgfs_read_info
1223};
1224
1225static ssize_t perf_dbgfs_read_run(struct file *filep, char __user *ubuf,
1226 size_t size, loff_t *offp)
635{ 1227{
636 struct pthr_ctx *pctx; 1228 struct perf_ctx *perf = filep->private_data;
637 int i; 1229 ssize_t ret, pos = 0;
1230 char *buf;
638 1231
639 for (i = 0; i < MAX_THREADS; i++) { 1232 buf = kmalloc(PERF_BUF_LEN, GFP_KERNEL);
640 pctx = &perf->pthr_ctx[i]; 1233 if (!buf)
641 if (pctx->thread) { 1234 return -ENOMEM;
642 pctx->status = kthread_stop(pctx->thread);
643 pctx->thread = NULL;
644 }
645 }
646}
647 1235
648static void perf_clear_thread_status(struct perf_ctx *perf) 1236 ret = perf_read_stats(perf, buf, PERF_BUF_LEN, &pos);
649{ 1237 if (ret)
650 int i; 1238 goto err_free;
1239
1240 ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
1241err_free:
1242 kfree(buf);
651 1243
652 for (i = 0; i < MAX_THREADS; i++) 1244 return ret;
653 perf->pthr_ctx[i].status = -ENODATA;
654} 1245}
655 1246
656static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf, 1247static ssize_t perf_dbgfs_write_run(struct file *filep, const char __user *ubuf,
657 size_t count, loff_t *offp) 1248 size_t size, loff_t *offp)
658{ 1249{
659 struct perf_ctx *perf = filp->private_data; 1250 struct perf_ctx *perf = filep->private_data;
660 int node, i; 1251 struct perf_peer *peer;
661 DECLARE_WAIT_QUEUE_HEAD(wq); 1252 int pidx, ret;
662 1253
663 if (wait_event_interruptible(perf->link_wq, perf->link_is_up)) 1254 ret = kstrtoint_from_user(ubuf, size, 0, &pidx);
664 return -ENOLINK; 1255 if (ret)
1256 return ret;
665 1257
666 if (perf->perf_threads == 0) 1258 if (pidx < 0 || pidx >= perf->pcnt)
667 return -EINVAL; 1259 return -EINVAL;
668 1260
669 if (!mutex_trylock(&perf->run_mutex)) 1261 peer = &perf->peers[pidx];
670 return -EBUSY;
671 1262
672 perf_clear_thread_status(perf); 1263 ret = perf_submit_test(peer);
1264 if (ret)
1265 return ret;
673 1266
674 if (perf->perf_threads > MAX_THREADS) { 1267 return size;
675 perf->perf_threads = MAX_THREADS; 1268}
676 pr_info("Reset total threads to: %u\n", MAX_THREADS);
677 }
678 1269
679 /* no greater than 1M */ 1270static const struct file_operations perf_dbgfs_run = {
680 if (seg_order > MAX_SEG_ORDER) { 1271 .open = simple_open,
681 seg_order = MAX_SEG_ORDER; 1272 .read = perf_dbgfs_read_run,
682 pr_info("Fix seg_order to %u\n", seg_order); 1273 .write = perf_dbgfs_write_run
683 } 1274};
684 1275
685 if (run_order < seg_order) { 1276static ssize_t perf_dbgfs_read_tcnt(struct file *filep, char __user *ubuf,
686 run_order = seg_order; 1277 size_t size, loff_t *offp)
687 pr_info("Fix run_order to %u\n", run_order); 1278{
688 } 1279 struct perf_ctx *perf = filep->private_data;
1280 char buf[8];
1281 ssize_t pos;
689 1282
690 node = on_node ? dev_to_node(&perf->ntb->pdev->dev) 1283 pos = scnprintf(buf, sizeof(buf), "%hhu\n", perf->tcnt);
691 : NUMA_NO_NODE;
692 atomic_set(&perf->tdone, 0);
693 1284
694 /* launch kernel thread */ 1285 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
695 for (i = 0; i < perf->perf_threads; i++) { 1286}
696 struct pthr_ctx *pctx;
697 1287
698 pctx = &perf->pthr_ctx[i]; 1288static ssize_t perf_dbgfs_write_tcnt(struct file *filep,
699 atomic_set(&pctx->dma_sync, 0); 1289 const char __user *ubuf,
700 pctx->perf = perf; 1290 size_t size, loff_t *offp)
701 pctx->wq = &wq; 1291{
702 pctx->thread = 1292 struct perf_ctx *perf = filep->private_data;
703 kthread_create_on_node(ntb_perf_thread, 1293 int ret;
704 (void *)pctx, 1294 u8 val;
705 node, "ntb_perf %d", i);
706 if (IS_ERR(pctx->thread)) {
707 pctx->thread = NULL;
708 goto err;
709 } else {
710 wake_up_process(pctx->thread);
711 }
712 }
713 1295
714 wait_event_interruptible(wq, 1296 ret = kstrtou8_from_user(ubuf, size, 0, &val);
715 atomic_read(&perf->tdone) == perf->perf_threads); 1297 if (ret)
1298 return ret;
716 1299
717 threads_cleanup(perf); 1300 ret = perf_set_tcnt(perf, val);
718 mutex_unlock(&perf->run_mutex); 1301 if (ret)
719 return count; 1302 return ret;
720 1303
721err: 1304 return size;
722 threads_cleanup(perf);
723 mutex_unlock(&perf->run_mutex);
724 return -ENXIO;
725} 1305}
726 1306
727static const struct file_operations ntb_perf_debugfs_run = { 1307static const struct file_operations perf_dbgfs_tcnt = {
728 .owner = THIS_MODULE,
729 .open = simple_open, 1308 .open = simple_open,
730 .read = debugfs_run_read, 1309 .read = perf_dbgfs_read_tcnt,
731 .write = debugfs_run_write, 1310 .write = perf_dbgfs_write_tcnt
732}; 1311};
733 1312
734static int perf_debugfs_setup(struct perf_ctx *perf) 1313static void perf_setup_dbgfs(struct perf_ctx *perf)
735{ 1314{
736 struct pci_dev *pdev = perf->ntb->pdev; 1315 struct pci_dev *pdev = perf->ntb->pdev;
737 struct dentry *debugfs_node_dir;
738 struct dentry *debugfs_run;
739 struct dentry *debugfs_threads;
740 struct dentry *debugfs_seg_order;
741 struct dentry *debugfs_run_order;
742 struct dentry *debugfs_use_dma;
743 struct dentry *debugfs_on_node;
744
745 if (!debugfs_initialized())
746 return -ENODEV;
747 1316
748 /* Assumpion: only one NTB device in the system */ 1317 perf->dbgfs_dir = debugfs_create_dir(pci_name(pdev), perf_dbgfs_topdir);
749 if (!perf_debugfs_dir) { 1318 if (!perf->dbgfs_dir) {
750 perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1319 dev_warn(&perf->ntb->dev, "DebugFS unsupported\n");
751 if (!perf_debugfs_dir) 1320 return;
752 return -ENODEV; 1321 }
753 } 1322
754 1323 debugfs_create_file("info", 0600, perf->dbgfs_dir, perf,
755 debugfs_node_dir = debugfs_create_dir(pci_name(pdev), 1324 &perf_dbgfs_info);
756 perf_debugfs_dir);
757 if (!debugfs_node_dir)
758 goto err;
759
760 debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR,
761 debugfs_node_dir, perf,
762 &ntb_perf_debugfs_run);
763 if (!debugfs_run)
764 goto err;
765
766 debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR,
767 debugfs_node_dir,
768 &perf->perf_threads);
769 if (!debugfs_threads)
770 goto err;
771
772 debugfs_seg_order = debugfs_create_u32("seg_order", 0600,
773 debugfs_node_dir,
774 &seg_order);
775 if (!debugfs_seg_order)
776 goto err;
777
778 debugfs_run_order = debugfs_create_u32("run_order", 0600,
779 debugfs_node_dir,
780 &run_order);
781 if (!debugfs_run_order)
782 goto err;
783
784 debugfs_use_dma = debugfs_create_bool("use_dma", 0600,
785 debugfs_node_dir,
786 &use_dma);
787 if (!debugfs_use_dma)
788 goto err;
789
790 debugfs_on_node = debugfs_create_bool("on_node", 0600,
791 debugfs_node_dir,
792 &on_node);
793 if (!debugfs_on_node)
794 goto err;
795 1325
796 return 0; 1326 debugfs_create_file("run", 0600, perf->dbgfs_dir, perf,
1327 &perf_dbgfs_run);
797 1328
798err: 1329 debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf,
799 debugfs_remove_recursive(perf_debugfs_dir); 1330 &perf_dbgfs_tcnt);
800 perf_debugfs_dir = NULL; 1331
801 return -ENODEV; 1332 /* They are made read-only for test exec safety and integrity */
1333 debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order);
1334
1335 debugfs_create_u8("total_order", 0500, perf->dbgfs_dir, &total_order);
1336
1337 debugfs_create_bool("use_dma", 0500, perf->dbgfs_dir, &use_dma);
802} 1338}
803 1339
804static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb) 1340static void perf_clear_dbgfs(struct perf_ctx *perf)
1341{
1342 debugfs_remove_recursive(perf->dbgfs_dir);
1343}
1344
1345/*==============================================================================
1346 * Basic driver initialization
1347 *==============================================================================
1348 */
1349
1350static struct perf_ctx *perf_create_data(struct ntb_dev *ntb)
805{ 1351{
806 struct pci_dev *pdev = ntb->pdev;
807 struct perf_ctx *perf; 1352 struct perf_ctx *perf;
808 int node;
809 int rc = 0;
810 1353
811 if (ntb_spad_count(ntb) < MAX_SPAD) { 1354 perf = devm_kzalloc(&ntb->dev, sizeof(*perf), GFP_KERNEL);
812 dev_err(&ntb->dev, "Not enough scratch pad registers for %s", 1355 if (!perf)
813 DRIVER_NAME); 1356 return ERR_PTR(-ENOMEM);
814 return -EIO;
815 }
816 1357
817 if (!ntb->ops->mw_set_trans) { 1358 perf->pcnt = ntb_peer_port_count(ntb);
818 dev_err(&ntb->dev, "Need inbound MW based NTB API\n"); 1359 perf->peers = devm_kcalloc(&ntb->dev, perf->pcnt, sizeof(*perf->peers),
819 return -EINVAL; 1360 GFP_KERNEL);
1361 if (!perf->peers)
1362 return ERR_PTR(-ENOMEM);
1363
1364 perf->ntb = ntb;
1365
1366 return perf;
1367}
1368
1369static int perf_setup_peer_mw(struct perf_peer *peer)
1370{
1371 struct perf_ctx *perf = peer->perf;
1372 phys_addr_t phys_addr;
1373 int ret;
1374
1375 /* Get outbound MW parameters and map it */
1376 ret = ntb_peer_mw_get_addr(perf->ntb, peer->gidx, &phys_addr,
1377 &peer->outbuf_size);
1378 if (ret)
1379 return ret;
1380
1381 peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,
1382 peer->outbuf_size);
1383 if (!peer->outbuf)
1384 return -ENOMEM;
1385
1386 if (max_mw_size && peer->outbuf_size > max_mw_size) {
1387 peer->outbuf_size = max_mw_size;
1388 dev_warn(&peer->perf->ntb->dev,
1389 "Peer %d outbuf reduced to %pa\n", peer->pidx,
1390 &peer->outbuf_size);
820 } 1391 }
821 1392
822 if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT) 1393 return 0;
823 dev_warn(&ntb->dev, "Multi-port NTB devices unsupported\n"); 1394}
824 1395
825 node = on_node ? dev_to_node(&pdev->dev) : NUMA_NO_NODE; 1396static int perf_init_peers(struct perf_ctx *perf)
826 perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node); 1397{
827 if (!perf) { 1398 struct perf_peer *peer;
828 rc = -ENOMEM; 1399 int pidx, lport, ret;
829 goto err_perf; 1400
1401 lport = ntb_port_number(perf->ntb);
1402 perf->gidx = -1;
1403 for (pidx = 0; pidx < perf->pcnt; pidx++) {
1404 peer = &perf->peers[pidx];
1405
1406 peer->perf = perf;
1407 peer->pidx = pidx;
1408 if (lport < ntb_peer_port_number(perf->ntb, pidx)) {
1409 if (perf->gidx == -1)
1410 perf->gidx = pidx;
1411 peer->gidx = pidx + 1;
1412 } else {
1413 peer->gidx = pidx;
1414 }
1415 INIT_WORK(&peer->service, perf_service_work);
830 } 1416 }
1417 if (perf->gidx == -1)
1418 perf->gidx = pidx;
831 1419
832 perf->ntb = ntb; 1420 for (pidx = 0; pidx < perf->pcnt; pidx++) {
833 perf->perf_threads = 1; 1421 ret = perf_setup_peer_mw(&perf->peers[pidx]);
834 atomic_set(&perf->tsync, 0); 1422 if (ret)
835 mutex_init(&perf->run_mutex); 1423 return ret;
836 spin_lock_init(&perf->db_lock); 1424 }
837 perf_setup_mw(ntb, perf); 1425
838 init_waitqueue_head(&perf->link_wq); 1426 dev_dbg(&perf->ntb->dev, "Global port index %d\n", perf->gidx);
839 INIT_DELAYED_WORK(&perf->link_work, perf_link_work); 1427
1428 return 0;
1429}
840 1430
841 rc = ntb_set_ctx(ntb, perf, &perf_ops); 1431static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
842 if (rc) 1432{
843 goto err_ctx; 1433 struct perf_ctx *perf;
1434 int ret;
844 1435
845 perf->link_is_up = false; 1436 perf = perf_create_data(ntb);
846 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1437 if (IS_ERR(perf))
847 ntb_link_event(ntb); 1438 return PTR_ERR(perf);
848 1439
849 rc = perf_debugfs_setup(perf); 1440 ret = perf_init_peers(perf);
850 if (rc) 1441 if (ret)
851 goto err_ctx; 1442 return ret;
852 1443
853 perf_clear_thread_status(perf); 1444 perf_init_threads(perf);
854 1445
855 return 0; 1446 ret = perf_init_service(perf);
1447 if (ret)
1448 return ret;
856 1449
857err_ctx: 1450 ret = perf_enable_service(perf);
858 cancel_delayed_work_sync(&perf->link_work); 1451 if (ret)
859 kfree(perf); 1452 return ret;
860err_perf: 1453
861 return rc; 1454 perf_setup_dbgfs(perf);
1455
1456 return 0;
862} 1457}
863 1458
864static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb) 1459static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
865{ 1460{
866 struct perf_ctx *perf = ntb->ctx; 1461 struct perf_ctx *perf = ntb->ctx;
867 int i;
868 1462
869 dev_dbg(&perf->ntb->dev, "%s called\n", __func__); 1463 perf_clear_dbgfs(perf);
870 1464
871 mutex_lock(&perf->run_mutex); 1465 perf_disable_service(perf);
872 1466
873 cancel_delayed_work_sync(&perf->link_work); 1467 perf_clear_threads(perf);
1468}
874 1469
875 ntb_clear_ctx(ntb); 1470static struct ntb_client perf_client = {
876 ntb_link_disable(ntb); 1471 .ops = {
1472 .probe = perf_probe,
1473 .remove = perf_remove
1474 }
1475};
877 1476
878 debugfs_remove_recursive(perf_debugfs_dir); 1477static int __init perf_init(void)
879 perf_debugfs_dir = NULL; 1478{
1479 int ret;
880 1480
881 if (use_dma) { 1481 if (chunk_order > MAX_CHUNK_ORDER) {
882 for (i = 0; i < MAX_THREADS; i++) { 1482 chunk_order = MAX_CHUNK_ORDER;
883 struct pthr_ctx *pctx = &perf->pthr_ctx[i]; 1483 pr_info("Chunk order reduced to %hhu\n", chunk_order);
1484 }
884 1485
885 if (pctx->dma_chan) 1486 if (total_order < chunk_order) {
886 dma_release_channel(pctx->dma_chan); 1487 total_order = chunk_order;
887 } 1488 pr_info("Total data order reduced to %hhu\n", total_order);
888 } 1489 }
889 1490
890 kfree(perf); 1491 perf_wq = alloc_workqueue("perf_wq", WQ_UNBOUND | WQ_SYSFS, 0);
1492 if (!perf_wq)
1493 return -ENOMEM;
1494
1495 if (debugfs_initialized())
1496 perf_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1497
1498 ret = ntb_register_client(&perf_client);
1499 if (ret) {
1500 debugfs_remove_recursive(perf_dbgfs_topdir);
1501 destroy_workqueue(perf_wq);
1502 }
1503
1504 return ret;
891} 1505}
1506module_init(perf_init);
1507
1508static void __exit perf_exit(void)
1509{
1510 ntb_unregister_client(&perf_client);
1511 debugfs_remove_recursive(perf_dbgfs_topdir);
1512 destroy_workqueue(perf_wq);
1513}
1514module_exit(perf_exit);
892 1515
893static struct ntb_client perf_client = {
894 .ops = {
895 .probe = perf_probe,
896 .remove = perf_remove,
897 },
898};
899module_ntb_client(perf_client);
diff --git a/drivers/ntb/test/ntb_pingpong.c b/drivers/ntb/test/ntb_pingpong.c
index 3f5a92bae6f8..65865e460ab8 100644
--- a/drivers/ntb/test/ntb_pingpong.c
+++ b/drivers/ntb/test/ntb_pingpong.c
@@ -1,10 +1,11 @@
1/* 1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or 2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license. 3 * redistributing this file, you may do so under either license.
4 * 4 *
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 7 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
8 * Copyright (C) 2017 T-Platforms. All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -18,6 +19,7 @@
18 * BSD LICENSE 19 * BSD LICENSE
19 * 20 *
20 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 21 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
22 * Copyright (C) 2017 T-Platforms. All Rights Reserved.
21 * 23 *
22 * Redistribution and use in source and binary forms, with or without 24 * Redistribution and use in source and binary forms, with or without
23 * modification, are permitted provided that the following conditions 25 * modification, are permitted provided that the following conditions
@@ -46,37 +48,45 @@
46 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 48 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 * 49 *
48 * PCIe NTB Pingpong Linux driver 50 * PCIe NTB Pingpong Linux driver
49 *
50 * Contact Information:
51 * Allen Hubbe <Allen.Hubbe@emc.com>
52 */ 51 */
53 52
54/* Note: load this module with option 'dyndbg=+p' */ 53/*
54 * How to use this tool, by example.
55 *
56 * Assuming $DBG_DIR is something like:
57 * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
58 * Suppose aside from local device there is at least one remote device
59 * connected to NTB with index 0.
60 *-----------------------------------------------------------------------------
61 * Eg: install driver with specified delay between doorbell event and response
62 *
63 * root@self# insmod ntb_pingpong.ko delay_ms=1000
64 *-----------------------------------------------------------------------------
65 * Eg: get number of ping-pong cycles performed
66 *
67 * root@self# cat $DBG_DIR/count
68 */
55 69
56#include <linux/init.h> 70#include <linux/init.h>
57#include <linux/kernel.h> 71#include <linux/kernel.h>
58#include <linux/module.h> 72#include <linux/module.h>
73#include <linux/device.h>
74#include <linux/bitops.h>
59 75
60#include <linux/dma-mapping.h>
61#include <linux/pci.h> 76#include <linux/pci.h>
62#include <linux/slab.h> 77#include <linux/slab.h>
63#include <linux/spinlock.h> 78#include <linux/hrtimer.h>
64#include <linux/debugfs.h> 79#include <linux/debugfs.h>
65 80
66#include <linux/ntb.h> 81#include <linux/ntb.h>
67 82
68#define DRIVER_NAME "ntb_pingpong" 83#define DRIVER_NAME "ntb_pingpong"
69#define DRIVER_DESCRIPTION "PCIe NTB Simple Pingpong Client" 84#define DRIVER_VERSION "2.0"
70
71#define DRIVER_LICENSE "Dual BSD/GPL"
72#define DRIVER_VERSION "1.0"
73#define DRIVER_RELDATE "24 March 2015"
74#define DRIVER_AUTHOR "Allen Hubbe <Allen.Hubbe@emc.com>"
75 85
76MODULE_LICENSE(DRIVER_LICENSE); 86MODULE_LICENSE("Dual BSD/GPL");
77MODULE_VERSION(DRIVER_VERSION); 87MODULE_VERSION(DRIVER_VERSION);
78MODULE_AUTHOR(DRIVER_AUTHOR); 88MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
79MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 89MODULE_DESCRIPTION("PCIe NTB Simple Pingpong Client");
80 90
81static unsigned int unsafe; 91static unsigned int unsafe;
82module_param(unsafe, uint, 0644); 92module_param(unsafe, uint, 0644);
@@ -86,237 +96,343 @@ static unsigned int delay_ms = 1000;
86module_param(delay_ms, uint, 0644); 96module_param(delay_ms, uint, 0644);
87MODULE_PARM_DESC(delay_ms, "Milliseconds to delay the response to peer"); 97MODULE_PARM_DESC(delay_ms, "Milliseconds to delay the response to peer");
88 98
89static unsigned long db_init = 0x7;
90module_param(db_init, ulong, 0644);
91MODULE_PARM_DESC(db_init, "Initial doorbell bits to ring on the peer");
92
93/* Only two-ports NTB devices are supported */
94#define PIDX NTB_DEF_PEER_IDX
95
96struct pp_ctx { 99struct pp_ctx {
97 struct ntb_dev *ntb; 100 struct ntb_dev *ntb;
98 u64 db_bits; 101 struct hrtimer timer;
99 /* synchronize access to db_bits by ping and pong */ 102 u64 in_db;
100 spinlock_t db_lock; 103 u64 out_db;
101 struct timer_list db_timer; 104 int out_pidx;
102 unsigned long db_delay; 105 u64 nmask;
103 struct dentry *debugfs_node_dir; 106 u64 pmask;
104 struct dentry *debugfs_count; 107 atomic_t count;
105 atomic_t count; 108 spinlock_t lock;
109 struct dentry *dbgfs_dir;
106}; 110};
111#define to_pp_timer(__timer) \
112 container_of(__timer, struct pp_ctx, timer)
107 113
108static struct dentry *pp_debugfs_dir; 114static struct dentry *pp_dbgfs_topdir;
109 115
110static void pp_ping(struct timer_list *t) 116static int pp_find_next_peer(struct pp_ctx *pp)
111{ 117{
112 struct pp_ctx *pp = from_timer(pp, t, db_timer); 118 u64 link, out_db;
113 unsigned long irqflags; 119 int pidx;
114 u64 db_bits, db_mask; 120
115 u32 spad_rd, spad_wr; 121 link = ntb_link_is_up(pp->ntb, NULL, NULL);
122
123 /* Find next available peer */
124 if (link & pp->nmask) {
125 pidx = __ffs64(link & pp->nmask);
126 out_db = BIT_ULL(pidx + 1);
127 } else if (link & pp->pmask) {
128 pidx = __ffs64(link & pp->pmask);
129 out_db = BIT_ULL(pidx);
130 } else {
131 return -ENODEV;
132 }
116 133
117 spin_lock_irqsave(&pp->db_lock, irqflags); 134 spin_lock(&pp->lock);
118 { 135 pp->out_pidx = pidx;
119 db_mask = ntb_db_valid_mask(pp->ntb); 136 pp->out_db = out_db;
120 db_bits = ntb_db_read(pp->ntb); 137 spin_unlock(&pp->lock);
121 138
122 if (db_bits) { 139 return 0;
123 dev_dbg(&pp->ntb->dev, 140}
124 "Masked pongs %#llx\n",
125 db_bits);
126 ntb_db_clear(pp->ntb, db_bits);
127 }
128 141
129 db_bits = ((pp->db_bits | db_bits) << 1) & db_mask; 142static void pp_setup(struct pp_ctx *pp)
143{
144 int ret;
130 145
131 if (!db_bits) 146 ntb_db_set_mask(pp->ntb, pp->in_db);
132 db_bits = db_init;
133 147
134 spad_rd = ntb_spad_read(pp->ntb, 0); 148 hrtimer_cancel(&pp->timer);
135 spad_wr = spad_rd + 1;
136 149
137 dev_dbg(&pp->ntb->dev, 150 ret = pp_find_next_peer(pp);
138 "Ping bits %#llx read %#x write %#x\n", 151 if (ret == -ENODEV) {
139 db_bits, spad_rd, spad_wr); 152 dev_dbg(&pp->ntb->dev, "Got no peers, so cancel\n");
153 return;
154 }
140 155
141 ntb_peer_spad_write(pp->ntb, PIDX, 0, spad_wr); 156 dev_dbg(&pp->ntb->dev, "Ping-pong started with port %d, db %#llx\n",
142 ntb_peer_db_set(pp->ntb, db_bits); 157 ntb_peer_port_number(pp->ntb, pp->out_pidx), pp->out_db);
143 ntb_db_clear_mask(pp->ntb, db_mask);
144 158
145 pp->db_bits = 0; 159 hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);
146 }
147 spin_unlock_irqrestore(&pp->db_lock, irqflags);
148} 160}
149 161
150static void pp_link_event(void *ctx) 162static void pp_clear(struct pp_ctx *pp)
151{ 163{
152 struct pp_ctx *pp = ctx; 164 hrtimer_cancel(&pp->timer);
153 165
154 if (ntb_link_is_up(pp->ntb, NULL, NULL) == 1) { 166 ntb_db_set_mask(pp->ntb, pp->in_db);
155 dev_dbg(&pp->ntb->dev, "link is up\n"); 167
156 pp_ping(&pp->db_timer); 168 dev_dbg(&pp->ntb->dev, "Ping-pong cancelled\n");
157 } else {
158 dev_dbg(&pp->ntb->dev, "link is down\n");
159 del_timer(&pp->db_timer);
160 }
161} 169}
162 170
163static void pp_db_event(void *ctx, int vec) 171static void pp_ping(struct pp_ctx *pp)
164{ 172{
165 struct pp_ctx *pp = ctx; 173 u32 count;
166 u64 db_bits, db_mask;
167 unsigned long irqflags;
168 174
169 spin_lock_irqsave(&pp->db_lock, irqflags); 175 count = atomic_read(&pp->count);
170 {
171 db_mask = ntb_db_vector_mask(pp->ntb, vec);
172 db_bits = db_mask & ntb_db_read(pp->ntb);
173 ntb_db_set_mask(pp->ntb, db_mask);
174 ntb_db_clear(pp->ntb, db_bits);
175 176
176 pp->db_bits |= db_bits; 177 spin_lock(&pp->lock);
178 ntb_peer_spad_write(pp->ntb, pp->out_pidx, 0, count);
179 ntb_peer_msg_write(pp->ntb, pp->out_pidx, 0, count);
177 180
178 mod_timer(&pp->db_timer, jiffies + pp->db_delay); 181 dev_dbg(&pp->ntb->dev, "Ping port %d spad %#x, msg %#x\n",
182 ntb_peer_port_number(pp->ntb, pp->out_pidx), count, count);
179 183
180 dev_dbg(&pp->ntb->dev, 184 ntb_peer_db_set(pp->ntb, pp->out_db);
181 "Pong vec %d bits %#llx\n", 185 ntb_db_clear_mask(pp->ntb, pp->in_db);
182 vec, db_bits); 186 spin_unlock(&pp->lock);
183 atomic_inc(&pp->count);
184 }
185 spin_unlock_irqrestore(&pp->db_lock, irqflags);
186} 187}
187 188
188static int pp_debugfs_setup(struct pp_ctx *pp) 189static void pp_pong(struct pp_ctx *pp)
189{ 190{
190 struct pci_dev *pdev = pp->ntb->pdev; 191 u32 msg_data = -1, spad_data = -1;
192 int pidx = 0;
191 193
192 if (!pp_debugfs_dir) 194 /* Read pong data */
193 return -ENODEV; 195 spad_data = ntb_spad_read(pp->ntb, 0);
196 msg_data = ntb_msg_read(pp->ntb, &pidx, 0);
197 ntb_msg_clear_sts(pp->ntb, -1);
194 198
195 pp->debugfs_node_dir = debugfs_create_dir(pci_name(pdev), 199 /*
196 pp_debugfs_dir); 200 * Scratchpad and message data may differ, since message register can't
197 if (!pp->debugfs_node_dir) 201 * be rewritten unless status is cleared. Additionally either of them
198 return -ENODEV; 202 * might be unsupported
203 */
204 dev_dbg(&pp->ntb->dev, "Pong spad %#x, msg %#x (port %d)\n",
205 spad_data, msg_data, ntb_peer_port_number(pp->ntb, pidx));
199 206
200 pp->debugfs_count = debugfs_create_atomic_t("count", S_IRUSR | S_IWUSR, 207 atomic_inc(&pp->count);
201 pp->debugfs_node_dir,
202 &pp->count);
203 if (!pp->debugfs_count)
204 return -ENODEV;
205 208
206 return 0; 209 ntb_db_set_mask(pp->ntb, pp->in_db);
210 ntb_db_clear(pp->ntb, pp->in_db);
211
212 hrtimer_start(&pp->timer, ms_to_ktime(delay_ms), HRTIMER_MODE_REL);
213}
214
215static enum hrtimer_restart pp_timer_func(struct hrtimer *t)
216{
217 struct pp_ctx *pp = to_pp_timer(t);
218
219 pp_ping(pp);
220
221 return HRTIMER_NORESTART;
222}
223
224static void pp_link_event(void *ctx)
225{
226 struct pp_ctx *pp = ctx;
227
228 pp_setup(pp);
229}
230
231static void pp_db_event(void *ctx, int vec)
232{
233 struct pp_ctx *pp = ctx;
234
235 pp_pong(pp);
207} 236}
208 237
209static const struct ntb_ctx_ops pp_ops = { 238static const struct ntb_ctx_ops pp_ops = {
210 .link_event = pp_link_event, 239 .link_event = pp_link_event,
211 .db_event = pp_db_event, 240 .db_event = pp_db_event
212}; 241};
213 242
214static int pp_probe(struct ntb_client *client, 243static int pp_check_ntb(struct ntb_dev *ntb)
215 struct ntb_dev *ntb)
216{ 244{
217 struct pp_ctx *pp; 245 u64 pmask;
218 int rc;
219 246
220 if (ntb_db_is_unsafe(ntb)) { 247 if (ntb_db_is_unsafe(ntb)) {
221 dev_dbg(&ntb->dev, "doorbell is unsafe\n"); 248 dev_dbg(&ntb->dev, "Doorbell is unsafe\n");
222 if (!unsafe) { 249 if (!unsafe)
223 rc = -EINVAL; 250 return -EINVAL;
224 goto err_pp;
225 }
226 }
227
228 if (ntb_spad_count(ntb) < 1) {
229 dev_dbg(&ntb->dev, "no enough scratchpads\n");
230 rc = -EINVAL;
231 goto err_pp;
232 } 251 }
233 252
234 if (ntb_spad_is_unsafe(ntb)) { 253 if (ntb_spad_is_unsafe(ntb)) {
235 dev_dbg(&ntb->dev, "scratchpad is unsafe\n"); 254 dev_dbg(&ntb->dev, "Scratchpad is unsafe\n");
236 if (!unsafe) { 255 if (!unsafe)
237 rc = -EINVAL; 256 return -EINVAL;
238 goto err_pp;
239 }
240 } 257 }
241 258
242 if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT) 259 pmask = GENMASK_ULL(ntb_peer_port_count(ntb), 0);
243 dev_warn(&ntb->dev, "multi-port NTB is unsupported\n"); 260 if ((ntb_db_valid_mask(ntb) & pmask) != pmask) {
261 dev_err(&ntb->dev, "Unsupported DB configuration\n");
262 return -EINVAL;
263 }
244 264
245 pp = kmalloc(sizeof(*pp), GFP_KERNEL); 265 if (ntb_spad_count(ntb) < 1 && ntb_msg_count(ntb) < 1) {
246 if (!pp) { 266 dev_err(&ntb->dev, "Scratchpads and messages unsupported\n");
247 rc = -ENOMEM; 267 return -EINVAL;
248 goto err_pp; 268 } else if (ntb_spad_count(ntb) < 1) {
269 dev_dbg(&ntb->dev, "Scratchpads unsupported\n");
270 } else if (ntb_msg_count(ntb) < 1) {
271 dev_dbg(&ntb->dev, "Messages unsupported\n");
249 } 272 }
250 273
274 return 0;
275}
276
277static struct pp_ctx *pp_create_data(struct ntb_dev *ntb)
278{
279 struct pp_ctx *pp;
280
281 pp = devm_kzalloc(&ntb->dev, sizeof(*pp), GFP_KERNEL);
282 if (!pp)
283 return ERR_PTR(-ENOMEM);
284
251 pp->ntb = ntb; 285 pp->ntb = ntb;
252 pp->db_bits = 0;
253 atomic_set(&pp->count, 0); 286 atomic_set(&pp->count, 0);
254 spin_lock_init(&pp->db_lock); 287 spin_lock_init(&pp->lock);
255 timer_setup(&pp->db_timer, pp_ping, 0); 288 hrtimer_init(&pp->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
256 pp->db_delay = msecs_to_jiffies(delay_ms); 289 pp->timer.function = pp_timer_func;
290
291 return pp;
292}
293
294static void pp_init_flds(struct pp_ctx *pp)
295{
296 int pidx, lport, pcnt;
297
298 /* Find global port index */
299 lport = ntb_port_number(pp->ntb);
300 pcnt = ntb_peer_port_count(pp->ntb);
301 for (pidx = 0; pidx < pcnt; pidx++) {
302 if (lport < ntb_peer_port_number(pp->ntb, pidx))
303 break;
304 }
257 305
258 rc = ntb_set_ctx(ntb, pp, &pp_ops); 306 pp->in_db = BIT_ULL(pidx);
259 if (rc) 307 pp->pmask = GENMASK_ULL(pidx, 0) >> 1;
260 goto err_ctx; 308 pp->nmask = GENMASK_ULL(pcnt - 1, pidx);
261 309
262 rc = pp_debugfs_setup(pp); 310 dev_dbg(&pp->ntb->dev, "Inbound db %#llx, prev %#llx, next %#llx\n",
263 if (rc) 311 pp->in_db, pp->pmask, pp->nmask);
264 goto err_ctx; 312}
313
314static int pp_mask_events(struct pp_ctx *pp)
315{
316 u64 db_mask, msg_mask;
317 int ret;
318
319 db_mask = ntb_db_valid_mask(pp->ntb);
320 ret = ntb_db_set_mask(pp->ntb, db_mask);
321 if (ret)
322 return ret;
265 323
266 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 324 /* Skip message events masking if unsupported */
267 ntb_link_event(ntb); 325 if (ntb_msg_count(pp->ntb) < 1)
326 return 0;
327
328 msg_mask = ntb_msg_outbits(pp->ntb) | ntb_msg_inbits(pp->ntb);
329 return ntb_msg_set_mask(pp->ntb, msg_mask);
330}
331
332static int pp_setup_ctx(struct pp_ctx *pp)
333{
334 int ret;
335
336 ret = ntb_set_ctx(pp->ntb, pp, &pp_ops);
337 if (ret)
338 return ret;
339
340 ntb_link_enable(pp->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
341 /* Might be not necessary */
342 ntb_link_event(pp->ntb);
268 343
269 return 0; 344 return 0;
345}
346
347static void pp_clear_ctx(struct pp_ctx *pp)
348{
349 ntb_link_disable(pp->ntb);
270 350
271err_ctx: 351 ntb_clear_ctx(pp->ntb);
272 kfree(pp);
273err_pp:
274 return rc;
275} 352}
276 353
277static void pp_remove(struct ntb_client *client, 354static void pp_setup_dbgfs(struct pp_ctx *pp)
278 struct ntb_dev *ntb) 355{
356 struct pci_dev *pdev = pp->ntb->pdev;
357 void *ret;
358
359 pp->dbgfs_dir = debugfs_create_dir(pci_name(pdev), pp_dbgfs_topdir);
360
361 ret = debugfs_create_atomic_t("count", 0600, pp->dbgfs_dir, &pp->count);
362 if (!ret)
363 dev_warn(&pp->ntb->dev, "DebugFS unsupported\n");
364}
365
366static void pp_clear_dbgfs(struct pp_ctx *pp)
367{
368 debugfs_remove_recursive(pp->dbgfs_dir);
369}
370
371static int pp_probe(struct ntb_client *client, struct ntb_dev *ntb)
372{
373 struct pp_ctx *pp;
374 int ret;
375
376 ret = pp_check_ntb(ntb);
377 if (ret)
378 return ret;
379
380 pp = pp_create_data(ntb);
381 if (IS_ERR(pp))
382 return PTR_ERR(pp);
383
384 pp_init_flds(pp);
385
386 ret = pp_mask_events(pp);
387 if (ret)
388 return ret;
389
390 ret = pp_setup_ctx(pp);
391 if (ret)
392 return ret;
393
394 pp_setup_dbgfs(pp);
395
396 return 0;
397}
398
399static void pp_remove(struct ntb_client *client, struct ntb_dev *ntb)
279{ 400{
280 struct pp_ctx *pp = ntb->ctx; 401 struct pp_ctx *pp = ntb->ctx;
281 402
282 debugfs_remove_recursive(pp->debugfs_node_dir); 403 pp_clear_dbgfs(pp);
283 404
284 ntb_clear_ctx(ntb); 405 pp_clear_ctx(pp);
285 del_timer_sync(&pp->db_timer);
286 ntb_link_disable(ntb);
287 406
288 kfree(pp); 407 pp_clear(pp);
289} 408}
290 409
291static struct ntb_client pp_client = { 410static struct ntb_client pp_client = {
292 .ops = { 411 .ops = {
293 .probe = pp_probe, 412 .probe = pp_probe,
294 .remove = pp_remove, 413 .remove = pp_remove
295 }, 414 }
296}; 415};
297 416
298static int __init pp_init(void) 417static int __init pp_init(void)
299{ 418{
300 int rc; 419 int ret;
301 420
302 if (debugfs_initialized()) 421 if (debugfs_initialized())
303 pp_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 422 pp_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
304 423
305 rc = ntb_register_client(&pp_client); 424 ret = ntb_register_client(&pp_client);
306 if (rc) 425 if (ret)
307 goto err_client; 426 debugfs_remove_recursive(pp_dbgfs_topdir);
308 427
309 return 0; 428 return ret;
310
311err_client:
312 debugfs_remove_recursive(pp_debugfs_dir);
313 return rc;
314} 429}
315module_init(pp_init); 430module_init(pp_init);
316 431
317static void __exit pp_exit(void) 432static void __exit pp_exit(void)
318{ 433{
319 ntb_unregister_client(&pp_client); 434 ntb_unregister_client(&pp_client);
320 debugfs_remove_recursive(pp_debugfs_dir); 435 debugfs_remove_recursive(pp_dbgfs_topdir);
321} 436}
322module_exit(pp_exit); 437module_exit(pp_exit);
438
diff --git a/drivers/ntb/test/ntb_tool.c b/drivers/ntb/test/ntb_tool.c
index 91526a986caa..d592c0ffbd19 100644
--- a/drivers/ntb/test/ntb_tool.c
+++ b/drivers/ntb/test/ntb_tool.c
@@ -5,6 +5,7 @@
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 7 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
8 * Copyright (C) 2017 T-Platforms All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -18,6 +19,7 @@
18 * BSD LICENSE 19 * BSD LICENSE
19 * 20 *
20 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 21 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
22 * Copyright (C) 2017 T-Platforms All Rights Reserved.
21 * 23 *
22 * Redistribution and use in source and binary forms, with or without 24 * Redistribution and use in source and binary forms, with or without
23 * modification, are permitted provided that the following conditions 25 * modification, are permitted provided that the following conditions
@@ -46,9 +48,6 @@
46 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 48 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47 * 49 *
48 * PCIe NTB Debugging Tool Linux driver 50 * PCIe NTB Debugging Tool Linux driver
49 *
50 * Contact Information:
51 * Allen Hubbe <Allen.Hubbe@emc.com>
52 */ 51 */
53 52
54/* 53/*
@@ -56,42 +55,125 @@
56 * 55 *
57 * Assuming $DBG_DIR is something like: 56 * Assuming $DBG_DIR is something like:
58 * '/sys/kernel/debug/ntb_tool/0000:00:03.0' 57 * '/sys/kernel/debug/ntb_tool/0000:00:03.0'
58 * Suppose aside from local device there is at least one remote device
59 * connected to NTB with index 0.
60 *-----------------------------------------------------------------------------
61 * Eg: check local/peer device information.
62 *
63 * # Get local device port number
64 * root@self# cat $DBG_DIR/port
65 *
66 * # Check local device functionality
67 * root@self# ls $DBG_DIR
68 * db msg1 msg_sts peer4/ port
69 * db_event msg2 peer0/ peer5/ spad0
70 * db_mask msg3 peer1/ peer_db spad1
71 * link msg_event peer2/ peer_db_mask spad2
72 * msg0 msg_mask peer3/ peer_spad spad3
73 * # As one can see it supports:
74 * # 1) four inbound message registers
75 * # 2) four inbound scratchpads
76 * # 3) up to six peer devices
77 *
78 * # Check peer device port number
79 * root@self# cat $DBG_DIR/peer0/port
80 *
81 * # Check peer device(s) functionality to be used
82 * root@self# ls $DBG_DIR/peer0
83 * link mw_trans0 mw_trans6 port
84 * link_event mw_trans1 mw_trans7 spad0
85 * msg0 mw_trans2 peer_mw_trans0 spad1
86 * msg1 mw_trans3 peer_mw_trans1 spad2
87 * msg2 mw_trans4 peer_mw_trans2 spad3
88 * msg3 mw_trans5 peer_mw_trans3
89 * # As one can see we got:
90 * # 1) four outbound message registers
91 * # 2) four outbound scratchpads
92 * # 3) eight inbound memory windows
93 * # 4) four outbound memory windows
94 *-----------------------------------------------------------------------------
95 * Eg: NTB link tests
59 * 96 *
60 * Eg: check if clearing the doorbell mask generates an interrupt. 97 * # Set local link up/down
98 * root@self# echo Y > $DBG_DIR/link
99 * root@self# echo N > $DBG_DIR/link
61 * 100 *
62 * # Check the link status 101 * # Check if link with peer device is up/down:
63 * root@self# cat $DBG_DIR/link 102 * root@self# cat $DBG_DIR/peer0/link
64 * 103 *
65 * # Block until the link is up 104 * # Block until the link is up/down
66 * root@self# echo Y > $DBG_DIR/link_event 105 * root@self# echo Y > $DBG_DIR/peer0/link_event
106 * root@self# echo N > $DBG_DIR/peer0/link_event
107 *-----------------------------------------------------------------------------
108 * Eg: Doorbell registers tests (some functionality might be absent)
67 * 109 *
68 * # Set the doorbell mask 110 * # Set/clear/get local doorbell
69 * root@self# echo 's 1' > $DBG_DIR/mask 111 * root@self# echo 's 1' > $DBG_DIR/db
112 * root@self# echo 'c 1' > $DBG_DIR/db
113 * root@self# cat $DBG_DIR/db
70 * 114 *
71 * # Ring the doorbell from the peer 115 * # Set/clear/get local doorbell mask
116 * root@self# echo 's 1' > $DBG_DIR/db_mask
117 * root@self# echo 'c 1' > $DBG_DIR/db_mask
118 * root@self# cat $DBG_DIR/db_mask
119 *
120 * # Ring/clear/get peer doorbell
72 * root@peer# echo 's 1' > $DBG_DIR/peer_db 121 * root@peer# echo 's 1' > $DBG_DIR/peer_db
122 * root@peer# echo 'c 1' > $DBG_DIR/peer_db
123 * root@peer# cat $DBG_DIR/peer_db
124 *
125 * # Set/clear/get peer doorbell mask
126 * root@self# echo 's 1' > $DBG_DIR/peer_db_mask
127 * root@self# echo 'c 1' > $DBG_DIR/peer_db_mask
128 * root@self# cat $DBG_DIR/peer_db_mask
129 *
130 * # Block until local doorbell is set with specified value
131 * root@self# echo 1 > $DBG_DIR/db_event
132 *-----------------------------------------------------------------------------
133 * Eg: Message registers tests (functionality might be absent)
73 * 134 *
74 * # Clear the doorbell mask 135 * # Set/clear/get in/out message registers status
75 * root@self# echo 'c 1' > $DBG_DIR/mask 136 * root@self# echo 's 1' > $DBG_DIR/msg_sts
137 * root@self# echo 'c 1' > $DBG_DIR/msg_sts
138 * root@self# cat $DBG_DIR/msg_sts
76 * 139 *
77 * Observe debugging output in dmesg or your console. You should see a 140 * # Set/clear in/out message registers mask
78 * doorbell event triggered by clearing the mask. If not, this may indicate an 141 * root@self# echo 's 1' > $DBG_DIR/msg_mask
79 * issue with the hardware that needs to be worked around in the driver. 142 * root@self# echo 'c 1' > $DBG_DIR/msg_mask
80 * 143 *
81 * Eg: read and write scratchpad registers 144 * # Get inbound message register #0 value and source of port index
145 * root@self# cat $DBG_DIR/msg0
82 * 146 *
83 * root@peer# echo '0 0x01010101 1 0x7f7f7f7f' > $DBG_DIR/peer_spad 147 * # Send some data to peer over outbound message register #0
148 * root@self# echo 0x01020304 > $DBG_DIR/peer0/msg0
149 *-----------------------------------------------------------------------------
150 * Eg: Scratchpad registers tests (functionality might be absent)
84 * 151 *
85 * root@self# cat $DBG_DIR/spad 152 * # Write/read to/from local scratchpad register #0
153 * root@peer# echo 0x01020304 > $DBG_DIR/spad0
154 * root@peer# cat $DBG_DIR/spad0
86 * 155 *
87 * Observe that spad 0 and 1 have the values set by the peer. 156 * # Write/read to/from peer scratchpad register #0
157 * root@peer# echo 0x01020304 > $DBG_DIR/peer0/spad0
158 * root@peer# cat $DBG_DIR/peer0/spad0
159 *-----------------------------------------------------------------------------
160 * Eg: Memory windows tests
88 * 161 *
89 * # Check the memory window translation info 162 * # Create inbound memory window buffer of specified size/get its base address
90 * cat $DBG_DIR/peer_trans0 163 * root@peer# echo 16384 > $DBG_DIR/peer0/mw_trans0
164 * root@peer# cat $DBG_DIR/peer0/mw_trans0
91 * 165 *
92 * # Setup a 16k memory window buffer 166 * # Write/read data to/from inbound memory window
93 * echo 16384 > $DBG_DIR/peer_trans0 167 * root@peer# echo Hello > $DBG_DIR/peer0/mw0
168 * root@peer# head -c 7 $DBG_DIR/peer0/mw0
94 * 169 *
170 * # Map outbound memory window/check it settings (on peer device)
171 * root@peer# echo 0xADD0BA5E:16384 > $DBG_DIR/peer0/peer_mw_trans0
172 * root@peer# cat $DBG_DIR/peer0/peer_mw_trans0
173 *
174 * # Write/read data to/from outbound memory window (on peer device)
175 * root@peer# echo olleH > $DBG_DIR/peer0/peer_mw0
176 * root@peer# head -c 7 $DBG_DIR/peer0/peer_mw0
95 */ 177 */
96 178
97#include <linux/init.h> 179#include <linux/init.h>
@@ -106,49 +188,87 @@
106 188
107#include <linux/ntb.h> 189#include <linux/ntb.h>
108 190
109#define DRIVER_NAME "ntb_tool" 191#define DRIVER_NAME "ntb_tool"
110#define DRIVER_DESCRIPTION "PCIe NTB Debugging Tool" 192#define DRIVER_VERSION "2.0"
111
112#define DRIVER_LICENSE "Dual BSD/GPL"
113#define DRIVER_VERSION "1.0"
114#define DRIVER_RELDATE "22 April 2015"
115#define DRIVER_AUTHOR "Allen Hubbe <Allen.Hubbe@emc.com>"
116 193
117MODULE_LICENSE(DRIVER_LICENSE); 194MODULE_LICENSE("Dual BSD/GPL");
118MODULE_VERSION(DRIVER_VERSION); 195MODULE_VERSION(DRIVER_VERSION);
119MODULE_AUTHOR(DRIVER_AUTHOR); 196MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
120MODULE_DESCRIPTION(DRIVER_DESCRIPTION); 197MODULE_DESCRIPTION("PCIe NTB Debugging Tool");
121
122/* It is rare to have hadrware with greater than six MWs */
123#define MAX_MWS 6
124/* Only two-ports devices are supported */
125#define PIDX NTB_DEF_PEER_IDX
126
127static struct dentry *tool_dbgfs;
128 198
199/*
200 * Inbound and outbound memory windows descriptor. Union members selection
201 * depends on the MW type the structure describes. mm_base/dma_base are the
202 * virtual and DMA address of an inbound MW. io_base/tr_base are the MMIO
203 * mapped virtual and xlat addresses of an outbound MW respectively.
204 */
129struct tool_mw { 205struct tool_mw {
130 int idx; 206 int widx;
207 int pidx;
131 struct tool_ctx *tc; 208 struct tool_ctx *tc;
132 resource_size_t win_size; 209 union {
210 u8 *mm_base;
211 u8 __iomem *io_base;
212 };
213 union {
214 dma_addr_t dma_base;
215 u64 tr_base;
216 };
133 resource_size_t size; 217 resource_size_t size;
134 u8 __iomem *local; 218 struct dentry *dbgfs_file;
135 u8 *peer; 219};
136 dma_addr_t peer_dma; 220
137 struct dentry *peer_dbg_file; 221/*
222 * Wrapper structure is used to distinguish the outbound MW peers reference
223 * within the corresponding DebugFS directory IO operation.
224 */
225struct tool_mw_wrap {
226 int pidx;
227 struct tool_mw *mw;
228};
229
230struct tool_msg {
231 int midx;
232 int pidx;
233 struct tool_ctx *tc;
234};
235
236struct tool_spad {
237 int sidx;
238 int pidx;
239 struct tool_ctx *tc;
240};
241
242struct tool_peer {
243 int pidx;
244 struct tool_ctx *tc;
245 int inmw_cnt;
246 struct tool_mw *inmws;
247 int outmw_cnt;
248 struct tool_mw_wrap *outmws;
249 int outmsg_cnt;
250 struct tool_msg *outmsgs;
251 int outspad_cnt;
252 struct tool_spad *outspads;
253 struct dentry *dbgfs_dir;
138}; 254};
139 255
140struct tool_ctx { 256struct tool_ctx {
141 struct ntb_dev *ntb; 257 struct ntb_dev *ntb;
142 struct dentry *dbgfs;
143 wait_queue_head_t link_wq; 258 wait_queue_head_t link_wq;
144 int mw_count; 259 wait_queue_head_t db_wq;
145 struct tool_mw mws[MAX_MWS]; 260 wait_queue_head_t msg_wq;
261 int outmw_cnt;
262 struct tool_mw *outmws;
263 int peer_cnt;
264 struct tool_peer *peers;
265 int inmsg_cnt;
266 struct tool_msg *inmsgs;
267 int inspad_cnt;
268 struct tool_spad *inspads;
269 struct dentry *dbgfs_dir;
146}; 270};
147 271
148#define SPAD_FNAME_SIZE 0x10
149#define INT_PTR(x) ((void *)(unsigned long)x)
150#define PTR_INT(x) ((int)(unsigned long)x)
151
152#define TOOL_FOPS_RDWR(__name, __read, __write) \ 272#define TOOL_FOPS_RDWR(__name, __read, __write) \
153 const struct file_operations __name = { \ 273 const struct file_operations __name = { \
154 .owner = THIS_MODULE, \ 274 .owner = THIS_MODULE, \
@@ -157,6 +277,15 @@ struct tool_ctx {
157 .write = __write, \ 277 .write = __write, \
158 } 278 }
159 279
280#define TOOL_BUF_LEN 32
281
282static struct dentry *tool_dbgfs_topdir;
283
284/*==============================================================================
285 * NTB events handlers
286 *==============================================================================
287 */
288
160static void tool_link_event(void *ctx) 289static void tool_link_event(void *ctx)
161{ 290{
162 struct tool_ctx *tc = ctx; 291 struct tool_ctx *tc = ctx;
@@ -182,580 +311,578 @@ static void tool_db_event(void *ctx, int vec)
182 311
183 dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n", 312 dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
184 vec, db_mask, db_bits); 313 vec, db_mask, db_bits);
314
315 wake_up(&tc->db_wq);
316}
317
318static void tool_msg_event(void *ctx)
319{
320 struct tool_ctx *tc = ctx;
321 u64 msg_sts;
322
323 msg_sts = ntb_msg_read_sts(tc->ntb);
324
325 dev_dbg(&tc->ntb->dev, "message bits %#llx\n", msg_sts);
326
327 wake_up(&tc->msg_wq);
185} 328}
186 329
187static const struct ntb_ctx_ops tool_ops = { 330static const struct ntb_ctx_ops tool_ops = {
188 .link_event = tool_link_event, 331 .link_event = tool_link_event,
189 .db_event = tool_db_event, 332 .db_event = tool_db_event,
333 .msg_event = tool_msg_event
190}; 334};
191 335
192static ssize_t tool_dbfn_read(struct tool_ctx *tc, char __user *ubuf, 336/*==============================================================================
193 size_t size, loff_t *offp, 337 * Common read/write methods
194 u64 (*db_read_fn)(struct ntb_dev *)) 338 *==============================================================================
339 */
340
341static ssize_t tool_fn_read(struct tool_ctx *tc, char __user *ubuf,
342 size_t size, loff_t *offp,
343 u64 (*fn_read)(struct ntb_dev *))
195{ 344{
196 size_t buf_size; 345 size_t buf_size;
197 char *buf; 346 char buf[TOOL_BUF_LEN];
198 ssize_t pos, rc; 347 ssize_t pos;
199 348
200 if (!db_read_fn) 349 if (!fn_read)
201 return -EINVAL; 350 return -EINVAL;
202 351
203 buf_size = min_t(size_t, size, 0x20); 352 buf_size = min(size, sizeof(buf));
204
205 buf = kmalloc(buf_size, GFP_KERNEL);
206 if (!buf)
207 return -ENOMEM;
208
209 pos = scnprintf(buf, buf_size, "%#llx\n",
210 db_read_fn(tc->ntb));
211 353
212 rc = simple_read_from_buffer(ubuf, size, offp, buf, pos); 354 pos = scnprintf(buf, buf_size, "%#llx\n", fn_read(tc->ntb));
213 355
214 kfree(buf); 356 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
215
216 return rc;
217} 357}
218 358
219static ssize_t tool_dbfn_write(struct tool_ctx *tc, 359static ssize_t tool_fn_write(struct tool_ctx *tc,
220 const char __user *ubuf, 360 const char __user *ubuf,
221 size_t size, loff_t *offp, 361 size_t size, loff_t *offp,
222 int (*db_set_fn)(struct ntb_dev *, u64), 362 int (*fn_set)(struct ntb_dev *, u64),
223 int (*db_clear_fn)(struct ntb_dev *, u64)) 363 int (*fn_clear)(struct ntb_dev *, u64))
224{ 364{
225 u64 db_bits;
226 char *buf, cmd; 365 char *buf, cmd;
227 ssize_t rc; 366 ssize_t ret;
367 u64 bits;
228 int n; 368 int n;
229 369
230 buf = kmalloc(size + 1, GFP_KERNEL); 370 buf = kmalloc(size + 1, GFP_KERNEL);
231 if (!buf) 371 if (!buf)
232 return -ENOMEM; 372 return -ENOMEM;
233 373
234 rc = simple_write_to_buffer(buf, size, offp, ubuf, size); 374 ret = simple_write_to_buffer(buf, size, offp, ubuf, size);
235 if (rc < 0) { 375 if (ret < 0) {
236 kfree(buf); 376 kfree(buf);
237 return rc; 377 return ret;
238 } 378 }
239 379
240 buf[size] = 0; 380 buf[size] = 0;
241 381
242 n = sscanf(buf, "%c %lli", &cmd, &db_bits); 382 n = sscanf(buf, "%c %lli", &cmd, &bits);
243 383
244 kfree(buf); 384 kfree(buf);
245 385
246 if (n != 2) { 386 if (n != 2) {
247 rc = -EINVAL; 387 ret = -EINVAL;
248 } else if (cmd == 's') { 388 } else if (cmd == 's') {
249 if (!db_set_fn) 389 if (!fn_set)
250 rc = -EINVAL; 390 ret = -EINVAL;
251 else 391 else
252 rc = db_set_fn(tc->ntb, db_bits); 392 ret = fn_set(tc->ntb, bits);
253 } else if (cmd == 'c') { 393 } else if (cmd == 'c') {
254 if (!db_clear_fn) 394 if (!fn_clear)
255 rc = -EINVAL; 395 ret = -EINVAL;
256 else 396 else
257 rc = db_clear_fn(tc->ntb, db_bits); 397 ret = fn_clear(tc->ntb, bits);
258 } else { 398 } else {
259 rc = -EINVAL; 399 ret = -EINVAL;
260 } 400 }
261 401
262 return rc ? : size; 402 return ret ? : size;
263} 403}
264 404
265static ssize_t tool_spadfn_read(struct tool_ctx *tc, char __user *ubuf, 405/*==============================================================================
266 size_t size, loff_t *offp, 406 * Port read/write methods
267 u32 (*spad_read_fn)(struct ntb_dev *, int)) 407 *==============================================================================
268{ 408 */
269 size_t buf_size;
270 char *buf;
271 ssize_t pos, rc;
272 int i, spad_count;
273
274 if (!spad_read_fn)
275 return -EINVAL;
276
277 spad_count = ntb_spad_count(tc->ntb);
278 409
279 /* 410static ssize_t tool_port_read(struct file *filep, char __user *ubuf,
280 * We multiply the number of spads by 15 to get the buffer size 411 size_t size, loff_t *offp)
281 * this is from 3 for the %d, 10 for the largest hex value 412{
282 * (0x00000000) and 2 for the tab and line feed. 413 struct tool_ctx *tc = filep->private_data;
283 */ 414 char buf[TOOL_BUF_LEN];
284 buf_size = min_t(size_t, size, spad_count * 15); 415 int pos;
285 416
286 buf = kmalloc(buf_size, GFP_KERNEL); 417 pos = scnprintf(buf, sizeof(buf), "%d\n", ntb_port_number(tc->ntb));
287 if (!buf)
288 return -ENOMEM;
289 418
290 pos = 0; 419 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
420}
291 421
292 for (i = 0; i < spad_count; ++i) { 422static TOOL_FOPS_RDWR(tool_port_fops,
293 pos += scnprintf(buf + pos, buf_size - pos, "%d\t%#x\n", 423 tool_port_read,
294 i, spad_read_fn(tc->ntb, i)); 424 NULL);
295 }
296 425
297 rc = simple_read_from_buffer(ubuf, size, offp, buf, pos); 426static ssize_t tool_peer_port_read(struct file *filep, char __user *ubuf,
427 size_t size, loff_t *offp)
428{
429 struct tool_peer *peer = filep->private_data;
430 struct tool_ctx *tc = peer->tc;
431 char buf[TOOL_BUF_LEN];
432 int pos;
298 433
299 kfree(buf); 434 pos = scnprintf(buf, sizeof(buf), "%d\n",
435 ntb_peer_port_number(tc->ntb, peer->pidx));
300 436
301 return rc; 437 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
302} 438}
303 439
304static ssize_t tool_spadfn_write(struct tool_ctx *tc, 440static TOOL_FOPS_RDWR(tool_peer_port_fops,
305 const char __user *ubuf, 441 tool_peer_port_read,
306 size_t size, loff_t *offp, 442 NULL);
307 int (*spad_write_fn)(struct ntb_dev *, 443
308 int, u32)) 444static int tool_init_peers(struct tool_ctx *tc)
309{ 445{
310 int spad_idx; 446 int pidx;
311 u32 spad_val;
312 char *buf, *buf_ptr;
313 int pos, n;
314 ssize_t rc;
315
316 if (!spad_write_fn) {
317 dev_dbg(&tc->ntb->dev, "no spad write fn\n");
318 return -EINVAL;
319 }
320 447
321 buf = kmalloc(size + 1, GFP_KERNEL); 448 tc->peer_cnt = ntb_peer_port_count(tc->ntb);
322 if (!buf) 449 tc->peers = devm_kcalloc(&tc->ntb->dev, tc->peer_cnt,
450 sizeof(*tc->peers), GFP_KERNEL);
451 if (tc->peers == NULL)
323 return -ENOMEM; 452 return -ENOMEM;
324 453
325 rc = simple_write_to_buffer(buf, size, offp, ubuf, size); 454 for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
326 if (rc < 0) { 455 tc->peers[pidx].pidx = pidx;
327 kfree(buf); 456 tc->peers[pidx].tc = tc;
328 return rc;
329 } 457 }
330 458
331 buf[size] = 0; 459 return 0;
332 buf_ptr = buf;
333 n = sscanf(buf_ptr, "%d %i%n", &spad_idx, &spad_val, &pos);
334 while (n == 2) {
335 buf_ptr += pos;
336 rc = spad_write_fn(tc->ntb, spad_idx, spad_val);
337 if (rc)
338 break;
339
340 n = sscanf(buf_ptr, "%d %i%n", &spad_idx, &spad_val, &pos);
341 }
342
343 if (n < 0)
344 rc = n;
345
346 kfree(buf);
347
348 return rc ? : size;
349} 460}
350 461
351static ssize_t tool_db_read(struct file *filep, char __user *ubuf, 462/*==============================================================================
352 size_t size, loff_t *offp) 463 * Link state read/write methods
353{ 464 *==============================================================================
354 struct tool_ctx *tc = filep->private_data; 465 */
355
356 return tool_dbfn_read(tc, ubuf, size, offp,
357 tc->ntb->ops->db_read);
358}
359 466
360static ssize_t tool_db_write(struct file *filep, const char __user *ubuf, 467static ssize_t tool_link_write(struct file *filep, const char __user *ubuf,
361 size_t size, loff_t *offp) 468 size_t size, loff_t *offp)
362{ 469{
363 struct tool_ctx *tc = filep->private_data; 470 struct tool_ctx *tc = filep->private_data;
471 bool val;
472 int ret;
364 473
365 return tool_dbfn_write(tc, ubuf, size, offp, 474 ret = kstrtobool_from_user(ubuf, size, &val);
366 tc->ntb->ops->db_set, 475 if (ret)
367 tc->ntb->ops->db_clear); 476 return ret;
368}
369 477
370static TOOL_FOPS_RDWR(tool_db_fops, 478 if (val)
371 tool_db_read, 479 ret = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
372 tool_db_write); 480 else
481 ret = ntb_link_disable(tc->ntb);
373 482
374static ssize_t tool_mask_read(struct file *filep, char __user *ubuf, 483 if (ret)
375 size_t size, loff_t *offp) 484 return ret;
376{
377 struct tool_ctx *tc = filep->private_data;
378 485
379 return tool_dbfn_read(tc, ubuf, size, offp, 486 return size;
380 tc->ntb->ops->db_read_mask);
381} 487}
382 488
383static ssize_t tool_mask_write(struct file *filep, const char __user *ubuf, 489static TOOL_FOPS_RDWR(tool_link_fops,
384 size_t size, loff_t *offp) 490 NULL,
491 tool_link_write);
492
493static ssize_t tool_peer_link_read(struct file *filep, char __user *ubuf,
494 size_t size, loff_t *offp)
385{ 495{
386 struct tool_ctx *tc = filep->private_data; 496 struct tool_peer *peer = filep->private_data;
497 struct tool_ctx *tc = peer->tc;
498 char buf[3];
387 499
388 return tool_dbfn_write(tc, ubuf, size, offp, 500 if (ntb_link_is_up(tc->ntb, NULL, NULL) & BIT(peer->pidx))
389 tc->ntb->ops->db_set_mask, 501 buf[0] = 'Y';
390 tc->ntb->ops->db_clear_mask); 502 else
503 buf[0] = 'N';
504 buf[1] = '\n';
505 buf[2] = '\0';
506
507 return simple_read_from_buffer(ubuf, size, offp, buf, 3);
391} 508}
392 509
393static TOOL_FOPS_RDWR(tool_mask_fops, 510static TOOL_FOPS_RDWR(tool_peer_link_fops,
394 tool_mask_read, 511 tool_peer_link_read,
395 tool_mask_write); 512 NULL);
396 513
397static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf, 514static ssize_t tool_peer_link_event_write(struct file *filep,
398 size_t size, loff_t *offp) 515 const char __user *ubuf,
516 size_t size, loff_t *offp)
399{ 517{
400 struct tool_ctx *tc = filep->private_data; 518 struct tool_peer *peer = filep->private_data;
519 struct tool_ctx *tc = peer->tc;
520 u64 link_msk;
521 bool val;
522 int ret;
401 523
402 return tool_dbfn_read(tc, ubuf, size, offp, 524 ret = kstrtobool_from_user(ubuf, size, &val);
403 tc->ntb->ops->peer_db_read); 525 if (ret)
404} 526 return ret;
405 527
406static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf, 528 link_msk = BIT_ULL_MASK(peer->pidx);
407 size_t size, loff_t *offp)
408{
409 struct tool_ctx *tc = filep->private_data;
410 529
411 return tool_dbfn_write(tc, ubuf, size, offp, 530 if (wait_event_interruptible(tc->link_wq,
412 tc->ntb->ops->peer_db_set, 531 !!(ntb_link_is_up(tc->ntb, NULL, NULL) & link_msk) == val))
413 tc->ntb->ops->peer_db_clear); 532 return -ERESTART;
533
534 return size;
414} 535}
415 536
416static TOOL_FOPS_RDWR(tool_peer_db_fops, 537static TOOL_FOPS_RDWR(tool_peer_link_event_fops,
417 tool_peer_db_read, 538 NULL,
418 tool_peer_db_write); 539 tool_peer_link_event_write);
419 540
420static ssize_t tool_peer_mask_read(struct file *filep, char __user *ubuf, 541/*==============================================================================
421 size_t size, loff_t *offp) 542 * Memory windows read/write/setting methods
543 *==============================================================================
544 */
545
546static ssize_t tool_mw_read(struct file *filep, char __user *ubuf,
547 size_t size, loff_t *offp)
422{ 548{
423 struct tool_ctx *tc = filep->private_data; 549 struct tool_mw *inmw = filep->private_data;
550
551 if (inmw->mm_base == NULL)
552 return -ENXIO;
424 553
425 return tool_dbfn_read(tc, ubuf, size, offp, 554 return simple_read_from_buffer(ubuf, size, offp,
426 tc->ntb->ops->peer_db_read_mask); 555 inmw->mm_base, inmw->size);
427} 556}
428 557
429static ssize_t tool_peer_mask_write(struct file *filep, const char __user *ubuf, 558static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf,
430 size_t size, loff_t *offp) 559 size_t size, loff_t *offp)
431{ 560{
432 struct tool_ctx *tc = filep->private_data; 561 struct tool_mw *inmw = filep->private_data;
433 562
434 return tool_dbfn_write(tc, ubuf, size, offp, 563 if (inmw->mm_base == NULL)
435 tc->ntb->ops->peer_db_set_mask, 564 return -ENXIO;
436 tc->ntb->ops->peer_db_clear_mask); 565
566 return simple_write_to_buffer(inmw->mm_base, inmw->size, offp,
567 ubuf, size);
437} 568}
438 569
439static TOOL_FOPS_RDWR(tool_peer_mask_fops, 570static TOOL_FOPS_RDWR(tool_mw_fops,
440 tool_peer_mask_read, 571 tool_mw_read,
441 tool_peer_mask_write); 572 tool_mw_write);
442 573
443static ssize_t tool_spad_read(struct file *filep, char __user *ubuf, 574static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
444 size_t size, loff_t *offp) 575 size_t req_size)
445{ 576{
446 struct tool_ctx *tc = filep->private_data; 577 resource_size_t size, addr_align, size_align;
578 struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
579 char buf[TOOL_BUF_LEN];
580 int ret;
447 581
448 return tool_spadfn_read(tc, ubuf, size, offp, 582 if (inmw->mm_base != NULL)
449 tc->ntb->ops->spad_read); 583 return 0;
450}
451 584
452static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf, 585 ret = ntb_mw_get_align(tc->ntb, pidx, widx, &addr_align,
453 size_t size, loff_t *offp) 586 &size_align, &size);
454{ 587 if (ret)
455 struct tool_ctx *tc = filep->private_data; 588 return ret;
589
590 inmw->size = min_t(resource_size_t, req_size, size);
591 inmw->size = round_up(inmw->size, addr_align);
592 inmw->size = round_up(inmw->size, size_align);
593 inmw->mm_base = dma_alloc_coherent(&tc->ntb->dev, inmw->size,
594 &inmw->dma_base, GFP_KERNEL);
595 if (!inmw->mm_base)
596 return -ENOMEM;
456 597
457 return tool_spadfn_write(tc, ubuf, size, offp, 598 if (!IS_ALIGNED(inmw->dma_base, addr_align)) {
458 tc->ntb->ops->spad_write); 599 ret = -ENOMEM;
459} 600 goto err_free_dma;
601 }
460 602
461static TOOL_FOPS_RDWR(tool_spad_fops, 603 ret = ntb_mw_set_trans(tc->ntb, pidx, widx, inmw->dma_base, inmw->size);
462 tool_spad_read, 604 if (ret)
463 tool_spad_write); 605 goto err_free_dma;
464 606
465static u32 ntb_tool_peer_spad_read(struct ntb_dev *ntb, int sidx) 607 snprintf(buf, sizeof(buf), "mw%d", widx);
466{ 608 inmw->dbgfs_file = debugfs_create_file(buf, 0600,
467 return ntb_peer_spad_read(ntb, PIDX, sidx); 609 tc->peers[pidx].dbgfs_dir, inmw,
468} 610 &tool_mw_fops);
469 611
470static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf, 612 return 0;
471 size_t size, loff_t *offp)
472{
473 struct tool_ctx *tc = filep->private_data;
474 613
475 return tool_spadfn_read(tc, ubuf, size, offp, ntb_tool_peer_spad_read); 614err_free_dma:
476} 615 dma_free_coherent(&tc->ntb->dev, inmw->size, inmw->mm_base,
616 inmw->dma_base);
617 inmw->mm_base = NULL;
618 inmw->dma_base = 0;
619 inmw->size = 0;
477 620
478static int ntb_tool_peer_spad_write(struct ntb_dev *ntb, int sidx, u32 val) 621 return ret;
479{
480 return ntb_peer_spad_write(ntb, PIDX, sidx, val);
481} 622}
482 623
483static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf, 624static void tool_free_mw(struct tool_ctx *tc, int pidx, int widx)
484 size_t size, loff_t *offp)
485{ 625{
486 struct tool_ctx *tc = filep->private_data; 626 struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
487 627
488 return tool_spadfn_write(tc, ubuf, size, offp, 628 debugfs_remove(inmw->dbgfs_file);
489 ntb_tool_peer_spad_write);
490}
491 629
492static TOOL_FOPS_RDWR(tool_peer_spad_fops, 630 if (inmw->mm_base != NULL) {
493 tool_peer_spad_read, 631 ntb_mw_clear_trans(tc->ntb, pidx, widx);
494 tool_peer_spad_write); 632 dma_free_coherent(&tc->ntb->dev, inmw->size,
495 633 inmw->mm_base, inmw->dma_base);
496static ssize_t tool_link_read(struct file *filep, char __user *ubuf, 634 }
497 size_t size, loff_t *offp)
498{
499 struct tool_ctx *tc = filep->private_data;
500 char buf[3];
501
502 buf[0] = ntb_link_is_up(tc->ntb, NULL, NULL) ? 'Y' : 'N';
503 buf[1] = '\n';
504 buf[2] = '\0';
505 635
506 return simple_read_from_buffer(ubuf, size, offp, buf, 2); 636 inmw->mm_base = NULL;
637 inmw->dma_base = 0;
638 inmw->size = 0;
639 inmw->dbgfs_file = NULL;
507} 640}
508 641
509static ssize_t tool_link_write(struct file *filep, const char __user *ubuf, 642static ssize_t tool_mw_trans_read(struct file *filep, char __user *ubuf,
510 size_t size, loff_t *offp) 643 size_t size, loff_t *offp)
511{ 644{
512 struct tool_ctx *tc = filep->private_data; 645 struct tool_mw *inmw = filep->private_data;
513 char buf[32]; 646 resource_size_t addr_align;
647 resource_size_t size_align;
648 resource_size_t size_max;
649 ssize_t ret, off = 0;
514 size_t buf_size; 650 size_t buf_size;
515 bool val; 651 char *buf;
516 int rc;
517 652
518 buf_size = min(size, (sizeof(buf) - 1)); 653 buf_size = min_t(size_t, size, 512);
519 if (copy_from_user(buf, ubuf, buf_size))
520 return -EFAULT;
521 654
522 buf[buf_size] = '\0'; 655 buf = kmalloc(buf_size, GFP_KERNEL);
656 if (!buf)
657 return -ENOMEM;
523 658
524 rc = strtobool(buf, &val); 659 ret = ntb_mw_get_align(inmw->tc->ntb, inmw->pidx, inmw->widx,
525 if (rc) 660 &addr_align, &size_align, &size_max);
526 return rc; 661 if (ret)
662 goto err;
527 663
528 if (val) 664 off += scnprintf(buf + off, buf_size - off,
529 rc = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 665 "Inbound MW \t%d\n",
530 else 666 inmw->widx);
531 rc = ntb_link_disable(tc->ntb);
532 667
533 if (rc) 668 off += scnprintf(buf + off, buf_size - off,
534 return rc; 669 "Port \t%d (%d)\n",
670 ntb_peer_port_number(inmw->tc->ntb, inmw->pidx),
671 inmw->pidx);
535 672
536 return size; 673 off += scnprintf(buf + off, buf_size - off,
537} 674 "Window Address \t0x%pK\n", inmw->mm_base);
538 675
539static TOOL_FOPS_RDWR(tool_link_fops, 676 off += scnprintf(buf + off, buf_size - off,
540 tool_link_read, 677 "DMA Address \t%pad\n",
541 tool_link_write); 678 &inmw->dma_base);
542 679
543static ssize_t tool_link_event_write(struct file *filep, 680 off += scnprintf(buf + off, buf_size - off,
544 const char __user *ubuf, 681 "Window Size \t%pa[p]\n",
545 size_t size, loff_t *offp) 682 &inmw->size);
546{
547 struct tool_ctx *tc = filep->private_data;
548 char buf[32];
549 size_t buf_size;
550 bool val;
551 int rc;
552 683
553 buf_size = min(size, (sizeof(buf) - 1)); 684 off += scnprintf(buf + off, buf_size - off,
554 if (copy_from_user(buf, ubuf, buf_size)) 685 "Alignment \t%pa[p]\n",
555 return -EFAULT; 686 &addr_align);
556 687
557 buf[buf_size] = '\0'; 688 off += scnprintf(buf + off, buf_size - off,
689 "Size Alignment \t%pa[p]\n",
690 &size_align);
558 691
559 rc = strtobool(buf, &val); 692 off += scnprintf(buf + off, buf_size - off,
560 if (rc) 693 "Size Max \t%pa[p]\n",
561 return rc; 694 &size_max);
562 695
563 if (wait_event_interruptible(tc->link_wq, 696 ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
564 ntb_link_is_up(tc->ntb, NULL, NULL) == val)) 697
565 return -ERESTART; 698err:
699 kfree(buf);
700
701 return ret;
702}
703
704static ssize_t tool_mw_trans_write(struct file *filep, const char __user *ubuf,
705 size_t size, loff_t *offp)
706{
707 struct tool_mw *inmw = filep->private_data;
708 unsigned int val;
709 int ret;
710
711 ret = kstrtouint_from_user(ubuf, size, 0, &val);
712 if (ret)
713 return ret;
714
715 tool_free_mw(inmw->tc, inmw->pidx, inmw->widx);
716 if (val) {
717 ret = tool_setup_mw(inmw->tc, inmw->pidx, inmw->widx, val);
718 if (ret)
719 return ret;
720 }
566 721
567 return size; 722 return size;
568} 723}
569 724
570static TOOL_FOPS_RDWR(tool_link_event_fops, 725static TOOL_FOPS_RDWR(tool_mw_trans_fops,
571 NULL, 726 tool_mw_trans_read,
572 tool_link_event_write); 727 tool_mw_trans_write);
573 728
574static ssize_t tool_mw_read(struct file *filep, char __user *ubuf, 729static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
575 size_t size, loff_t *offp) 730 size_t size, loff_t *offp)
576{ 731{
577 struct tool_mw *mw = filep->private_data; 732 struct tool_mw *outmw = filep->private_data;
578 ssize_t rc;
579 loff_t pos = *offp; 733 loff_t pos = *offp;
734 ssize_t ret;
580 void *buf; 735 void *buf;
581 736
582 if (mw->local == NULL) 737 if (outmw->io_base == NULL)
583 return -EIO; 738 return -EIO;
584 if (pos < 0) 739
585 return -EINVAL; 740 if (pos >= outmw->size || !size)
586 if (pos >= mw->win_size || !size)
587 return 0; 741 return 0;
588 if (size > mw->win_size - pos) 742
589 size = mw->win_size - pos; 743 if (size > outmw->size - pos)
744 size = outmw->size - pos;
590 745
591 buf = kmalloc(size, GFP_KERNEL); 746 buf = kmalloc(size, GFP_KERNEL);
592 if (!buf) 747 if (!buf)
593 return -ENOMEM; 748 return -ENOMEM;
594 749
595 memcpy_fromio(buf, mw->local + pos, size); 750 memcpy_fromio(buf, outmw->io_base + pos, size);
596 rc = copy_to_user(ubuf, buf, size); 751 ret = copy_to_user(ubuf, buf, size);
597 if (rc == size) { 752 if (ret == size) {
598 rc = -EFAULT; 753 ret = -EFAULT;
599 goto err_free; 754 goto err_free;
600 } 755 }
601 756
602 size -= rc; 757 size -= ret;
603 *offp = pos + size; 758 *offp = pos + size;
604 rc = size; 759 ret = size;
605 760
606err_free: 761err_free:
607 kfree(buf); 762 kfree(buf);
608 763
609 return rc; 764 return ret;
610} 765}
611 766
612static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf, 767static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
613 size_t size, loff_t *offp) 768 size_t size, loff_t *offp)
614{ 769{
615 struct tool_mw *mw = filep->private_data; 770 struct tool_mw *outmw = filep->private_data;
616 ssize_t rc; 771 ssize_t ret;
617 loff_t pos = *offp; 772 loff_t pos = *offp;
618 void *buf; 773 void *buf;
619 774
620 if (pos < 0) 775 if (outmw->io_base == NULL)
621 return -EINVAL; 776 return -EIO;
622 if (pos >= mw->win_size || !size) 777
778 if (pos >= outmw->size || !size)
623 return 0; 779 return 0;
624 if (size > mw->win_size - pos) 780 if (size > outmw->size - pos)
625 size = mw->win_size - pos; 781 size = outmw->size - pos;
626 782
627 buf = kmalloc(size, GFP_KERNEL); 783 buf = kmalloc(size, GFP_KERNEL);
628 if (!buf) 784 if (!buf)
629 return -ENOMEM; 785 return -ENOMEM;
630 786
631 rc = copy_from_user(buf, ubuf, size); 787 ret = copy_from_user(buf, ubuf, size);
632 if (rc == size) { 788 if (ret == size) {
633 rc = -EFAULT; 789 ret = -EFAULT;
634 goto err_free; 790 goto err_free;
635 } 791 }
636 792
637 size -= rc; 793 size -= ret;
638 *offp = pos + size; 794 *offp = pos + size;
639 rc = size; 795 ret = size;
640 796
641 memcpy_toio(mw->local + pos, buf, size); 797 memcpy_toio(outmw->io_base + pos, buf, size);
642 798
643err_free: 799err_free:
644 kfree(buf); 800 kfree(buf);
645 801
646 return rc; 802 return ret;
647}
648
649static TOOL_FOPS_RDWR(tool_mw_fops,
650 tool_mw_read,
651 tool_mw_write);
652
653static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
654 size_t size, loff_t *offp)
655{
656 struct tool_mw *mw = filep->private_data;
657
658 if (!mw->peer)
659 return -ENXIO;
660
661 return simple_read_from_buffer(ubuf, size, offp, mw->peer, mw->size);
662}
663
664static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
665 size_t size, loff_t *offp)
666{
667 struct tool_mw *mw = filep->private_data;
668
669 if (!mw->peer)
670 return -ENXIO;
671
672 return simple_write_to_buffer(mw->peer, mw->size, offp, ubuf, size);
673} 803}
674 804
675static TOOL_FOPS_RDWR(tool_peer_mw_fops, 805static TOOL_FOPS_RDWR(tool_peer_mw_fops,
676 tool_peer_mw_read, 806 tool_peer_mw_read,
677 tool_peer_mw_write); 807 tool_peer_mw_write);
678 808
679static int tool_setup_mw(struct tool_ctx *tc, int idx, size_t req_size) 809static int tool_setup_peer_mw(struct tool_ctx *tc, int pidx, int widx,
810 u64 req_addr, size_t req_size)
680{ 811{
681 int rc; 812 struct tool_mw *outmw = &tc->outmws[widx];
682 struct tool_mw *mw = &tc->mws[idx]; 813 resource_size_t map_size;
683 resource_size_t size, align_addr, align_size; 814 phys_addr_t map_base;
684 char buf[16]; 815 char buf[TOOL_BUF_LEN];
816 int ret;
685 817
686 if (mw->peer) 818 if (outmw->io_base != NULL)
687 return 0; 819 return 0;
688 820
689 rc = ntb_mw_get_align(tc->ntb, PIDX, idx, &align_addr, 821 ret = ntb_peer_mw_get_addr(tc->ntb, widx, &map_base, &map_size);
690 &align_size, &size); 822 if (ret)
691 if (rc) 823 return ret;
692 return rc;
693 824
694 mw->size = min_t(resource_size_t, req_size, size); 825 ret = ntb_peer_mw_set_trans(tc->ntb, pidx, widx, req_addr, req_size);
695 mw->size = round_up(mw->size, align_addr); 826 if (ret)
696 mw->size = round_up(mw->size, align_size); 827 return ret;
697 mw->peer = dma_alloc_coherent(&tc->ntb->pdev->dev, mw->size,
698 &mw->peer_dma, GFP_KERNEL);
699 828
700 if (!mw->peer || !IS_ALIGNED(mw->peer_dma, align_addr)) 829 outmw->io_base = ioremap_wc(map_base, map_size);
701 return -ENOMEM; 830 if (outmw->io_base == NULL) {
831 ret = -EFAULT;
832 goto err_clear_trans;
833 }
702 834
703 rc = ntb_mw_set_trans(tc->ntb, PIDX, idx, mw->peer_dma, mw->size); 835 outmw->tr_base = req_addr;
704 if (rc) 836 outmw->size = req_size;
705 goto err_free_dma; 837 outmw->pidx = pidx;
706 838
707 snprintf(buf, sizeof(buf), "peer_mw%d", idx); 839 snprintf(buf, sizeof(buf), "peer_mw%d", widx);
708 mw->peer_dbg_file = debugfs_create_file(buf, S_IRUSR | S_IWUSR, 840 outmw->dbgfs_file = debugfs_create_file(buf, 0600,
709 mw->tc->dbgfs, mw, 841 tc->peers[pidx].dbgfs_dir, outmw,
710 &tool_peer_mw_fops); 842 &tool_peer_mw_fops);
711 843
712 return 0; 844 return 0;
713 845
714err_free_dma: 846err_clear_trans:
715 dma_free_coherent(&tc->ntb->pdev->dev, mw->size, 847 ntb_peer_mw_clear_trans(tc->ntb, pidx, widx);
716 mw->peer, 848
717 mw->peer_dma); 849 return ret;
718 mw->peer = NULL;
719 mw->peer_dma = 0;
720 mw->size = 0;
721
722 return rc;
723} 850}
724 851
725static void tool_free_mw(struct tool_ctx *tc, int idx) 852static void tool_free_peer_mw(struct tool_ctx *tc, int widx)
726{ 853{
727 struct tool_mw *mw = &tc->mws[idx]; 854 struct tool_mw *outmw = &tc->outmws[widx];
728
729 if (mw->peer) {
730 ntb_mw_clear_trans(tc->ntb, PIDX, idx);
731 dma_free_coherent(&tc->ntb->pdev->dev, mw->size,
732 mw->peer,
733 mw->peer_dma);
734 }
735 855
736 mw->peer = NULL; 856 debugfs_remove(outmw->dbgfs_file);
737 mw->peer_dma = 0;
738 857
739 debugfs_remove(mw->peer_dbg_file); 858 if (outmw->io_base != NULL) {
859 iounmap(tc->outmws[widx].io_base);
860 ntb_peer_mw_clear_trans(tc->ntb, outmw->pidx, widx);
861 }
740 862
741 mw->peer_dbg_file = NULL; 863 outmw->io_base = NULL;
864 outmw->tr_base = 0;
865 outmw->size = 0;
866 outmw->pidx = -1;
867 outmw->dbgfs_file = NULL;
742} 868}
743 869
744static ssize_t tool_peer_mw_trans_read(struct file *filep, 870static ssize_t tool_peer_mw_trans_read(struct file *filep, char __user *ubuf,
745 char __user *ubuf, 871 size_t size, loff_t *offp)
746 size_t size, loff_t *offp)
747{ 872{
748 struct tool_mw *mw = filep->private_data; 873 struct tool_mw_wrap *outmw_wrap = filep->private_data;
749 874 struct tool_mw *outmw = outmw_wrap->mw;
750 char *buf; 875 resource_size_t map_size;
876 phys_addr_t map_base;
877 ssize_t off = 0;
751 size_t buf_size; 878 size_t buf_size;
752 ssize_t ret, off = 0; 879 char *buf;
880 int ret;
753 881
754 phys_addr_t base; 882 ret = ntb_peer_mw_get_addr(outmw->tc->ntb, outmw->widx,
755 resource_size_t mw_size; 883 &map_base, &map_size);
756 resource_size_t align_addr = 0; 884 if (ret)
757 resource_size_t align_size = 0; 885 return ret;
758 resource_size_t max_size = 0;
759 886
760 buf_size = min_t(size_t, size, 512); 887 buf_size = min_t(size_t, size, 512);
761 888
@@ -763,43 +890,37 @@ static ssize_t tool_peer_mw_trans_read(struct file *filep,
763 if (!buf) 890 if (!buf)
764 return -ENOMEM; 891 return -ENOMEM;
765 892
766 ntb_mw_get_align(mw->tc->ntb, PIDX, mw->idx,
767 &align_addr, &align_size, &max_size);
768 ntb_peer_mw_get_addr(mw->tc->ntb, mw->idx, &base, &mw_size);
769
770 off += scnprintf(buf + off, buf_size - off, 893 off += scnprintf(buf + off, buf_size - off,
771 "Peer MW %d Information:\n", mw->idx); 894 "Outbound MW: \t%d\n", outmw->widx);
772 895
773 off += scnprintf(buf + off, buf_size - off, 896 if (outmw->io_base != NULL) {
774 "Physical Address \t%pa[p]\n", 897 off += scnprintf(buf + off, buf_size - off,
775 &base); 898 "Port attached \t%d (%d)\n",
776 899 ntb_peer_port_number(outmw->tc->ntb, outmw->pidx),
777 off += scnprintf(buf + off, buf_size - off, 900 outmw->pidx);
778 "Window Size \t%lld\n", 901 } else {
779 (unsigned long long)mw_size); 902 off += scnprintf(buf + off, buf_size - off,
903 "Port attached \t-1 (-1)\n");
904 }
780 905
781 off += scnprintf(buf + off, buf_size - off, 906 off += scnprintf(buf + off, buf_size - off,
782 "Alignment \t%lld\n", 907 "Virtual address \t0x%pK\n", outmw->io_base);
783 (unsigned long long)align_addr);
784 908
785 off += scnprintf(buf + off, buf_size - off, 909 off += scnprintf(buf + off, buf_size - off,
786 "Size Alignment \t%lld\n", 910 "Phys Address \t%pa[p]\n", &map_base);
787 (unsigned long long)align_size);
788 911
789 off += scnprintf(buf + off, buf_size - off, 912 off += scnprintf(buf + off, buf_size - off,
790 "Size Max \t%lld\n", 913 "Mapping Size \t%pa[p]\n", &map_size);
791 (unsigned long long)max_size);
792 914
793 off += scnprintf(buf + off, buf_size - off, 915 off += scnprintf(buf + off, buf_size - off,
794 "Ready \t%c\n", 916 "Translation Address \t0x%016llx\n", outmw->tr_base);
795 (mw->peer) ? 'Y' : 'N');
796 917
797 off += scnprintf(buf + off, buf_size - off, 918 off += scnprintf(buf + off, buf_size - off,
798 "Allocated Size \t%zd\n", 919 "Window Size \t%pa[p]\n", &outmw->size);
799 (mw->peer) ? (size_t)mw->size : 0);
800 920
801 ret = simple_read_from_buffer(ubuf, size, offp, buf, off); 921 ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
802 kfree(buf); 922 kfree(buf);
923
803 return ret; 924 return ret;
804} 925}
805 926
@@ -807,12 +928,12 @@ static ssize_t tool_peer_mw_trans_write(struct file *filep,
807 const char __user *ubuf, 928 const char __user *ubuf,
808 size_t size, loff_t *offp) 929 size_t size, loff_t *offp)
809{ 930{
810 struct tool_mw *mw = filep->private_data; 931 struct tool_mw_wrap *outmw_wrap = filep->private_data;
811 932 struct tool_mw *outmw = outmw_wrap->mw;
812 char buf[32]; 933 size_t buf_size, wsize;
813 size_t buf_size; 934 char buf[TOOL_BUF_LEN];
814 unsigned long long val; 935 int ret, n;
815 int rc; 936 u64 addr;
816 937
817 buf_size = min(size, (sizeof(buf) - 1)); 938 buf_size = min(size, (sizeof(buf) - 1));
818 if (copy_from_user(buf, ubuf, buf_size)) 939 if (copy_from_user(buf, ubuf, buf_size))
@@ -820,16 +941,17 @@ static ssize_t tool_peer_mw_trans_write(struct file *filep,
820 941
821 buf[buf_size] = '\0'; 942 buf[buf_size] = '\0';
822 943
823 rc = kstrtoull(buf, 0, &val); 944 n = sscanf(buf, "%lli:%zi", &addr, &wsize);
824 if (rc) 945 if (n != 2)
825 return rc; 946 return -EINVAL;
826
827 tool_free_mw(mw->tc, mw->idx);
828 if (val)
829 rc = tool_setup_mw(mw->tc, mw->idx, val);
830 947
831 if (rc) 948 tool_free_peer_mw(outmw->tc, outmw->widx);
832 return rc; 949 if (wsize) {
950 ret = tool_setup_peer_mw(outmw->tc, outmw_wrap->pidx,
951 outmw->widx, addr, wsize);
952 if (ret)
953 return ret;
954 }
833 955
834 return size; 956 return size;
835} 957}
@@ -838,195 +960,734 @@ static TOOL_FOPS_RDWR(tool_peer_mw_trans_fops,
838 tool_peer_mw_trans_read, 960 tool_peer_mw_trans_read,
839 tool_peer_mw_trans_write); 961 tool_peer_mw_trans_write);
840 962
841static int tool_init_mw(struct tool_ctx *tc, int idx) 963static int tool_init_mws(struct tool_ctx *tc)
842{ 964{
843 struct tool_mw *mw = &tc->mws[idx]; 965 int widx, pidx;
844 phys_addr_t base; 966
845 int rc; 967 /* Initialize outbound memory windows */
846 968 tc->outmw_cnt = ntb_peer_mw_count(tc->ntb);
847 rc = ntb_peer_mw_get_addr(tc->ntb, idx, &base, &mw->win_size); 969 tc->outmws = devm_kcalloc(&tc->ntb->dev, tc->outmw_cnt,
848 if (rc) 970 sizeof(*tc->outmws), GFP_KERNEL);
849 return rc; 971 if (tc->outmws == NULL)
850 972 return -ENOMEM;
851 mw->tc = tc; 973
852 mw->idx = idx; 974 for (widx = 0; widx < tc->outmw_cnt; widx++) {
853 mw->local = ioremap_wc(base, mw->win_size); 975 tc->outmws[widx].widx = widx;
854 if (!mw->local) 976 tc->outmws[widx].pidx = -1;
855 return -EFAULT; 977 tc->outmws[widx].tc = tc;
978 }
979
980 /* Initialize inbound memory windows and outbound MWs wrapper */
981 for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
982 tc->peers[pidx].inmw_cnt = ntb_mw_count(tc->ntb, pidx);
983 tc->peers[pidx].inmws =
984 devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].inmw_cnt,
985 sizeof(*tc->peers[pidx].inmws), GFP_KERNEL);
986 if (tc->peers[pidx].inmws == NULL)
987 return -ENOMEM;
988
989 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
990 tc->peers[pidx].inmws[widx].widx = widx;
991 tc->peers[pidx].inmws[widx].pidx = pidx;
992 tc->peers[pidx].inmws[widx].tc = tc;
993 }
994
995 tc->peers[pidx].outmw_cnt = ntb_peer_mw_count(tc->ntb);
996 tc->peers[pidx].outmws =
997 devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmw_cnt,
998 sizeof(*tc->peers[pidx].outmws), GFP_KERNEL);
999
1000 for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
1001 tc->peers[pidx].outmws[widx].pidx = pidx;
1002 tc->peers[pidx].outmws[widx].mw = &tc->outmws[widx];
1003 }
1004 }
856 1005
857 return 0; 1006 return 0;
858} 1007}
859 1008
860static void tool_free_mws(struct tool_ctx *tc) 1009static void tool_clear_mws(struct tool_ctx *tc)
861{ 1010{
862 int i; 1011 int widx, pidx;
1012
1013 /* Free outbound memory windows */
1014 for (widx = 0; widx < tc->outmw_cnt; widx++)
1015 tool_free_peer_mw(tc, widx);
863 1016
864 for (i = 0; i < tc->mw_count; i++) { 1017 /* Free outbound memory windows */
865 tool_free_mw(tc, i); 1018 for (pidx = 0; pidx < tc->peer_cnt; pidx++)
1019 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++)
1020 tool_free_mw(tc, pidx, widx);
1021}
866 1022
867 if (tc->mws[i].local) 1023/*==============================================================================
868 iounmap(tc->mws[i].local); 1024 * Doorbell read/write methods
1025 *==============================================================================
1026 */
869 1027
870 tc->mws[i].local = NULL; 1028static ssize_t tool_db_read(struct file *filep, char __user *ubuf,
871 } 1029 size_t size, loff_t *offp)
1030{
1031 struct tool_ctx *tc = filep->private_data;
1032
1033 return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read);
872} 1034}
873 1035
874static void tool_setup_dbgfs(struct tool_ctx *tc) 1036static ssize_t tool_db_write(struct file *filep, const char __user *ubuf,
1037 size_t size, loff_t *offp)
875{ 1038{
876 int i; 1039 struct tool_ctx *tc = filep->private_data;
877 1040
878 /* This modules is useless without dbgfs... */ 1041 return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set,
879 if (!tool_dbgfs) { 1042 tc->ntb->ops->db_clear);
880 tc->dbgfs = NULL; 1043}
881 return; 1044
1045static TOOL_FOPS_RDWR(tool_db_fops,
1046 tool_db_read,
1047 tool_db_write);
1048
1049static ssize_t tool_db_valid_mask_read(struct file *filep, char __user *ubuf,
1050 size_t size, loff_t *offp)
1051{
1052 struct tool_ctx *tc = filep->private_data;
1053
1054 return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_valid_mask);
1055}
1056
1057static TOOL_FOPS_RDWR(tool_db_valid_mask_fops,
1058 tool_db_valid_mask_read,
1059 NULL);
1060
1061static ssize_t tool_db_mask_read(struct file *filep, char __user *ubuf,
1062 size_t size, loff_t *offp)
1063{
1064 struct tool_ctx *tc = filep->private_data;
1065
1066 return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read_mask);
1067}
1068
1069static ssize_t tool_db_mask_write(struct file *filep, const char __user *ubuf,
1070 size_t size, loff_t *offp)
1071{
1072 struct tool_ctx *tc = filep->private_data;
1073
1074 return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set_mask,
1075 tc->ntb->ops->db_clear_mask);
1076}
1077
1078static TOOL_FOPS_RDWR(tool_db_mask_fops,
1079 tool_db_mask_read,
1080 tool_db_mask_write);
1081
1082static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf,
1083 size_t size, loff_t *offp)
1084{
1085 struct tool_ctx *tc = filep->private_data;
1086
1087 return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->peer_db_read);
1088}
1089
1090static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf,
1091 size_t size, loff_t *offp)
1092{
1093 struct tool_ctx *tc = filep->private_data;
1094
1095 return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->peer_db_set,
1096 tc->ntb->ops->peer_db_clear);
1097}
1098
1099static TOOL_FOPS_RDWR(tool_peer_db_fops,
1100 tool_peer_db_read,
1101 tool_peer_db_write);
1102
1103static ssize_t tool_peer_db_mask_read(struct file *filep, char __user *ubuf,
1104 size_t size, loff_t *offp)
1105{
1106 struct tool_ctx *tc = filep->private_data;
1107
1108 return tool_fn_read(tc, ubuf, size, offp,
1109 tc->ntb->ops->peer_db_read_mask);
1110}
1111
1112static ssize_t tool_peer_db_mask_write(struct file *filep,
1113 const char __user *ubuf,
1114 size_t size, loff_t *offp)
1115{
1116 struct tool_ctx *tc = filep->private_data;
1117
1118 return tool_fn_write(tc, ubuf, size, offp,
1119 tc->ntb->ops->peer_db_set_mask,
1120 tc->ntb->ops->peer_db_clear_mask);
1121}
1122
1123static TOOL_FOPS_RDWR(tool_peer_db_mask_fops,
1124 tool_peer_db_mask_read,
1125 tool_peer_db_mask_write);
1126
1127static ssize_t tool_db_event_write(struct file *filep,
1128 const char __user *ubuf,
1129 size_t size, loff_t *offp)
1130{
1131 struct tool_ctx *tc = filep->private_data;
1132 u64 val;
1133 int ret;
1134
1135 ret = kstrtou64_from_user(ubuf, size, 0, &val);
1136 if (ret)
1137 return ret;
1138
1139 if (wait_event_interruptible(tc->db_wq, ntb_db_read(tc->ntb) == val))
1140 return -ERESTART;
1141
1142 return size;
1143}
1144
1145static TOOL_FOPS_RDWR(tool_db_event_fops,
1146 NULL,
1147 tool_db_event_write);
1148
1149/*==============================================================================
1150 * Scratchpads read/write methods
1151 *==============================================================================
1152 */
1153
1154static ssize_t tool_spad_read(struct file *filep, char __user *ubuf,
1155 size_t size, loff_t *offp)
1156{
1157 struct tool_spad *spad = filep->private_data;
1158 char buf[TOOL_BUF_LEN];
1159 ssize_t pos;
1160
1161 if (!spad->tc->ntb->ops->spad_read)
1162 return -EINVAL;
1163
1164 pos = scnprintf(buf, sizeof(buf), "%#x\n",
1165 ntb_spad_read(spad->tc->ntb, spad->sidx));
1166
1167 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1168}
1169
1170static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
1171 size_t size, loff_t *offp)
1172{
1173 struct tool_spad *spad = filep->private_data;
1174 u32 val;
1175 int ret;
1176
1177 if (!spad->tc->ntb->ops->spad_write) {
1178 dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1179 return -EINVAL;
882 } 1180 }
883 1181
884 tc->dbgfs = debugfs_create_dir(dev_name(&tc->ntb->dev), 1182 ret = kstrtou32_from_user(ubuf, size, 0, &val);
885 tool_dbgfs); 1183 if (ret)
886 if (!tc->dbgfs) 1184 return ret;
887 return;
888 1185
889 debugfs_create_file("db", S_IRUSR | S_IWUSR, tc->dbgfs, 1186 ret = ntb_spad_write(spad->tc->ntb, spad->sidx, val);
890 tc, &tool_db_fops);
891 1187
892 debugfs_create_file("mask", S_IRUSR | S_IWUSR, tc->dbgfs, 1188 return ret ?: size;
893 tc, &tool_mask_fops); 1189}
894 1190
895 debugfs_create_file("peer_db", S_IRUSR | S_IWUSR, tc->dbgfs, 1191static TOOL_FOPS_RDWR(tool_spad_fops,
896 tc, &tool_peer_db_fops); 1192 tool_spad_read,
1193 tool_spad_write);
897 1194
898 debugfs_create_file("peer_mask", S_IRUSR | S_IWUSR, tc->dbgfs, 1195static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf,
899 tc, &tool_peer_mask_fops); 1196 size_t size, loff_t *offp)
1197{
1198 struct tool_spad *spad = filep->private_data;
1199 char buf[TOOL_BUF_LEN];
1200 ssize_t pos;
900 1201
901 debugfs_create_file("spad", S_IRUSR | S_IWUSR, tc->dbgfs, 1202 if (!spad->tc->ntb->ops->peer_spad_read)
902 tc, &tool_spad_fops); 1203 return -EINVAL;
903 1204
904 debugfs_create_file("peer_spad", S_IRUSR | S_IWUSR, tc->dbgfs, 1205 pos = scnprintf(buf, sizeof(buf), "%#x\n",
905 tc, &tool_peer_spad_fops); 1206 ntb_peer_spad_read(spad->tc->ntb, spad->pidx, spad->sidx));
906 1207
907 debugfs_create_file("link", S_IRUSR | S_IWUSR, tc->dbgfs, 1208 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
908 tc, &tool_link_fops); 1209}
909 1210
910 debugfs_create_file("link_event", S_IWUSR, tc->dbgfs, 1211static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf,
911 tc, &tool_link_event_fops); 1212 size_t size, loff_t *offp)
1213{
1214 struct tool_spad *spad = filep->private_data;
1215 u32 val;
1216 int ret;
1217
1218 if (!spad->tc->ntb->ops->peer_spad_write) {
1219 dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1220 return -EINVAL;
1221 }
1222
1223 ret = kstrtou32_from_user(ubuf, size, 0, &val);
1224 if (ret)
1225 return ret;
1226
1227 ret = ntb_peer_spad_write(spad->tc->ntb, spad->pidx, spad->sidx, val);
1228
1229 return ret ?: size;
1230}
912 1231
913 for (i = 0; i < tc->mw_count; i++) { 1232static TOOL_FOPS_RDWR(tool_peer_spad_fops,
914 char buf[30]; 1233 tool_peer_spad_read,
1234 tool_peer_spad_write);
915 1235
916 snprintf(buf, sizeof(buf), "mw%d", i); 1236static int tool_init_spads(struct tool_ctx *tc)
917 debugfs_create_file(buf, S_IRUSR | S_IWUSR, tc->dbgfs, 1237{
918 &tc->mws[i], &tool_mw_fops); 1238 int sidx, pidx;
1239
1240 /* Initialize inbound scratchpad structures */
1241 tc->inspad_cnt = ntb_spad_count(tc->ntb);
1242 tc->inspads = devm_kcalloc(&tc->ntb->dev, tc->inspad_cnt,
1243 sizeof(*tc->inspads), GFP_KERNEL);
1244 if (tc->inspads == NULL)
1245 return -ENOMEM;
1246
1247 for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1248 tc->inspads[sidx].sidx = sidx;
1249 tc->inspads[sidx].pidx = -1;
1250 tc->inspads[sidx].tc = tc;
1251 }
919 1252
920 snprintf(buf, sizeof(buf), "peer_trans%d", i); 1253 /* Initialize outbound scratchpad structures */
921 debugfs_create_file(buf, S_IRUSR | S_IWUSR, tc->dbgfs, 1254 for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
922 &tc->mws[i], &tool_peer_mw_trans_fops); 1255 tc->peers[pidx].outspad_cnt = ntb_spad_count(tc->ntb);
1256 tc->peers[pidx].outspads =
1257 devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outspad_cnt,
1258 sizeof(*tc->peers[pidx].outspads), GFP_KERNEL);
1259 if (tc->peers[pidx].outspads == NULL)
1260 return -ENOMEM;
1261
1262 for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1263 tc->peers[pidx].outspads[sidx].sidx = sidx;
1264 tc->peers[pidx].outspads[sidx].pidx = pidx;
1265 tc->peers[pidx].outspads[sidx].tc = tc;
1266 }
923 } 1267 }
1268
1269 return 0;
924} 1270}
925 1271
926static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb) 1272/*==============================================================================
1273 * Messages read/write methods
1274 *==============================================================================
1275 */
1276
1277static ssize_t tool_inmsg_read(struct file *filep, char __user *ubuf,
1278 size_t size, loff_t *offp)
927{ 1279{
928 struct tool_ctx *tc; 1280 struct tool_msg *msg = filep->private_data;
929 int rc; 1281 char buf[TOOL_BUF_LEN];
930 int i; 1282 ssize_t pos;
1283 u32 data;
1284 int pidx;
1285
1286 data = ntb_msg_read(msg->tc->ntb, &pidx, msg->midx);
1287
1288 pos = scnprintf(buf, sizeof(buf), "0x%08x<-%d\n", data, pidx);
1289
1290 return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1291}
1292
1293static TOOL_FOPS_RDWR(tool_inmsg_fops,
1294 tool_inmsg_read,
1295 NULL);
1296
1297static ssize_t tool_outmsg_write(struct file *filep,
1298 const char __user *ubuf,
1299 size_t size, loff_t *offp)
1300{
1301 struct tool_msg *msg = filep->private_data;
1302 u32 val;
1303 int ret;
1304
1305 ret = kstrtou32_from_user(ubuf, size, 0, &val);
1306 if (ret)
1307 return ret;
1308
1309 ret = ntb_peer_msg_write(msg->tc->ntb, msg->pidx, msg->midx, val);
1310
1311 return ret ? : size;
1312}
1313
1314static TOOL_FOPS_RDWR(tool_outmsg_fops,
1315 NULL,
1316 tool_outmsg_write);
1317
1318static ssize_t tool_msg_sts_read(struct file *filep, char __user *ubuf,
1319 size_t size, loff_t *offp)
1320{
1321 struct tool_ctx *tc = filep->private_data;
1322
1323 return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_read_sts);
1324}
1325
1326static ssize_t tool_msg_sts_write(struct file *filep, const char __user *ubuf,
1327 size_t size, loff_t *offp)
1328{
1329 struct tool_ctx *tc = filep->private_data;
1330
1331 return tool_fn_write(tc, ubuf, size, offp, NULL,
1332 tc->ntb->ops->msg_clear_sts);
1333}
1334
1335static TOOL_FOPS_RDWR(tool_msg_sts_fops,
1336 tool_msg_sts_read,
1337 tool_msg_sts_write);
1338
1339static ssize_t tool_msg_inbits_read(struct file *filep, char __user *ubuf,
1340 size_t size, loff_t *offp)
1341{
1342 struct tool_ctx *tc = filep->private_data;
1343
1344 return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_inbits);
1345}
1346
1347static TOOL_FOPS_RDWR(tool_msg_inbits_fops,
1348 tool_msg_inbits_read,
1349 NULL);
931 1350
932 if (!ntb->ops->mw_set_trans) { 1351static ssize_t tool_msg_outbits_read(struct file *filep, char __user *ubuf,
933 dev_dbg(&ntb->dev, "need inbound MW based NTB API\n"); 1352 size_t size, loff_t *offp)
934 rc = -EINVAL; 1353{
935 goto err_tc; 1354 struct tool_ctx *tc = filep->private_data;
1355
1356 return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_outbits);
1357}
1358
1359static TOOL_FOPS_RDWR(tool_msg_outbits_fops,
1360 tool_msg_outbits_read,
1361 NULL);
1362
1363static ssize_t tool_msg_mask_write(struct file *filep, const char __user *ubuf,
1364 size_t size, loff_t *offp)
1365{
1366 struct tool_ctx *tc = filep->private_data;
1367
1368 return tool_fn_write(tc, ubuf, size, offp,
1369 tc->ntb->ops->msg_set_mask,
1370 tc->ntb->ops->msg_clear_mask);
1371}
1372
1373static TOOL_FOPS_RDWR(tool_msg_mask_fops,
1374 NULL,
1375 tool_msg_mask_write);
1376
1377static ssize_t tool_msg_event_write(struct file *filep,
1378 const char __user *ubuf,
1379 size_t size, loff_t *offp)
1380{
1381 struct tool_ctx *tc = filep->private_data;
1382 u64 val;
1383 int ret;
1384
1385 ret = kstrtou64_from_user(ubuf, size, 0, &val);
1386 if (ret)
1387 return ret;
1388
1389 if (wait_event_interruptible(tc->msg_wq,
1390 ntb_msg_read_sts(tc->ntb) == val))
1391 return -ERESTART;
1392
1393 return size;
1394}
1395
1396static TOOL_FOPS_RDWR(tool_msg_event_fops,
1397 NULL,
1398 tool_msg_event_write);
1399
1400static int tool_init_msgs(struct tool_ctx *tc)
1401{
1402 int midx, pidx;
1403
1404 /* Initialize inbound message structures */
1405 tc->inmsg_cnt = ntb_msg_count(tc->ntb);
1406 tc->inmsgs = devm_kcalloc(&tc->ntb->dev, tc->inmsg_cnt,
1407 sizeof(*tc->inmsgs), GFP_KERNEL);
1408 if (tc->inmsgs == NULL)
1409 return -ENOMEM;
1410
1411 for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1412 tc->inmsgs[midx].midx = midx;
1413 tc->inmsgs[midx].pidx = -1;
1414 tc->inmsgs[midx].tc = tc;
936 } 1415 }
937 1416
938 if (ntb_spad_count(ntb) < 1) { 1417 /* Initialize outbound message structures */
939 dev_dbg(&ntb->dev, "no enough scratchpads\n"); 1418 for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
940 rc = -EINVAL; 1419 tc->peers[pidx].outmsg_cnt = ntb_msg_count(tc->ntb);
941 goto err_tc; 1420 tc->peers[pidx].outmsgs =
1421 devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmsg_cnt,
1422 sizeof(*tc->peers[pidx].outmsgs), GFP_KERNEL);
1423 if (tc->peers[pidx].outmsgs == NULL)
1424 return -ENOMEM;
1425
1426 for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1427 tc->peers[pidx].outmsgs[midx].midx = midx;
1428 tc->peers[pidx].outmsgs[midx].pidx = pidx;
1429 tc->peers[pidx].outmsgs[midx].tc = tc;
1430 }
942 } 1431 }
943 1432
1433 return 0;
1434}
1435
1436/*==============================================================================
1437 * Initialization methods
1438 *==============================================================================
1439 */
1440
1441static struct tool_ctx *tool_create_data(struct ntb_dev *ntb)
1442{
1443 struct tool_ctx *tc;
1444
1445 tc = devm_kzalloc(&ntb->dev, sizeof(*tc), GFP_KERNEL);
1446 if (tc == NULL)
1447 return ERR_PTR(-ENOMEM);
1448
1449 tc->ntb = ntb;
1450 init_waitqueue_head(&tc->link_wq);
1451 init_waitqueue_head(&tc->db_wq);
1452 init_waitqueue_head(&tc->msg_wq);
1453
944 if (ntb_db_is_unsafe(ntb)) 1454 if (ntb_db_is_unsafe(ntb))
945 dev_dbg(&ntb->dev, "doorbell is unsafe\n"); 1455 dev_dbg(&ntb->dev, "doorbell is unsafe\n");
946 1456
947 if (ntb_spad_is_unsafe(ntb)) 1457 if (ntb_spad_is_unsafe(ntb))
948 dev_dbg(&ntb->dev, "scratchpad is unsafe\n"); 1458 dev_dbg(&ntb->dev, "scratchpad is unsafe\n");
949 1459
950 if (ntb_peer_port_count(ntb) != NTB_DEF_PEER_CNT) 1460 return tc;
951 dev_warn(&ntb->dev, "multi-port NTB is unsupported\n"); 1461}
1462
1463static void tool_clear_data(struct tool_ctx *tc)
1464{
1465 wake_up(&tc->link_wq);
1466 wake_up(&tc->db_wq);
1467 wake_up(&tc->msg_wq);
1468}
1469
1470static int tool_init_ntb(struct tool_ctx *tc)
1471{
1472 return ntb_set_ctx(tc->ntb, tc, &tool_ops);
1473}
952 1474
953 tc = kzalloc(sizeof(*tc), GFP_KERNEL); 1475static void tool_clear_ntb(struct tool_ctx *tc)
954 if (!tc) { 1476{
955 rc = -ENOMEM; 1477 ntb_clear_ctx(tc->ntb);
956 goto err_tc; 1478 ntb_link_disable(tc->ntb);
1479}
1480
1481static void tool_setup_dbgfs(struct tool_ctx *tc)
1482{
1483 int pidx, widx, sidx, midx;
1484 char buf[TOOL_BUF_LEN];
1485
1486 /* This modules is useless without dbgfs... */
1487 if (!tool_dbgfs_topdir) {
1488 tc->dbgfs_dir = NULL;
1489 return;
957 } 1490 }
958 1491
959 tc->ntb = ntb; 1492 tc->dbgfs_dir = debugfs_create_dir(dev_name(&tc->ntb->dev),
960 init_waitqueue_head(&tc->link_wq); 1493 tool_dbgfs_topdir);
1494 if (!tc->dbgfs_dir)
1495 return;
1496
1497 debugfs_create_file("port", 0600, tc->dbgfs_dir,
1498 tc, &tool_port_fops);
1499
1500 debugfs_create_file("link", 0600, tc->dbgfs_dir,
1501 tc, &tool_link_fops);
1502
1503 debugfs_create_file("db", 0600, tc->dbgfs_dir,
1504 tc, &tool_db_fops);
1505
1506 debugfs_create_file("db_valid_mask", 0600, tc->dbgfs_dir,
1507 tc, &tool_db_valid_mask_fops);
1508
1509 debugfs_create_file("db_mask", 0600, tc->dbgfs_dir,
1510 tc, &tool_db_mask_fops);
961 1511
962 tc->mw_count = min(ntb_peer_mw_count(tc->ntb), MAX_MWS); 1512 debugfs_create_file("db_event", 0600, tc->dbgfs_dir,
963 for (i = 0; i < tc->mw_count; i++) { 1513 tc, &tool_db_event_fops);
964 rc = tool_init_mw(tc, i); 1514
965 if (rc) 1515 debugfs_create_file("peer_db", 0600, tc->dbgfs_dir,
966 goto err_ctx; 1516 tc, &tool_peer_db_fops);
1517
1518 debugfs_create_file("peer_db_mask", 0600, tc->dbgfs_dir,
1519 tc, &tool_peer_db_mask_fops);
1520
1521 if (tc->inspad_cnt != 0) {
1522 for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1523 snprintf(buf, sizeof(buf), "spad%d", sidx);
1524
1525 debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1526 &tc->inspads[sidx], &tool_spad_fops);
1527 }
967 } 1528 }
968 1529
969 tool_setup_dbgfs(tc); 1530 if (tc->inmsg_cnt != 0) {
1531 for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1532 snprintf(buf, sizeof(buf), "msg%d", midx);
1533 debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1534 &tc->inmsgs[midx], &tool_inmsg_fops);
1535 }
1536
1537 debugfs_create_file("msg_sts", 0600, tc->dbgfs_dir,
1538 tc, &tool_msg_sts_fops);
1539
1540 debugfs_create_file("msg_inbits", 0600, tc->dbgfs_dir,
1541 tc, &tool_msg_inbits_fops);
970 1542
971 rc = ntb_set_ctx(ntb, tc, &tool_ops); 1543 debugfs_create_file("msg_outbits", 0600, tc->dbgfs_dir,
972 if (rc) 1544 tc, &tool_msg_outbits_fops);
973 goto err_ctx;
974 1545
975 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1546 debugfs_create_file("msg_mask", 0600, tc->dbgfs_dir,
976 ntb_link_event(ntb); 1547 tc, &tool_msg_mask_fops);
1548
1549 debugfs_create_file("msg_event", 0600, tc->dbgfs_dir,
1550 tc, &tool_msg_event_fops);
1551 }
1552
1553 for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1554 snprintf(buf, sizeof(buf), "peer%d", pidx);
1555 tc->peers[pidx].dbgfs_dir =
1556 debugfs_create_dir(buf, tc->dbgfs_dir);
1557
1558 debugfs_create_file("port", 0600,
1559 tc->peers[pidx].dbgfs_dir,
1560 &tc->peers[pidx], &tool_peer_port_fops);
1561
1562 debugfs_create_file("link", 0200,
1563 tc->peers[pidx].dbgfs_dir,
1564 &tc->peers[pidx], &tool_peer_link_fops);
1565
1566 debugfs_create_file("link_event", 0200,
1567 tc->peers[pidx].dbgfs_dir,
1568 &tc->peers[pidx], &tool_peer_link_event_fops);
1569
1570 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
1571 snprintf(buf, sizeof(buf), "mw_trans%d", widx);
1572 debugfs_create_file(buf, 0600,
1573 tc->peers[pidx].dbgfs_dir,
1574 &tc->peers[pidx].inmws[widx],
1575 &tool_mw_trans_fops);
1576 }
1577
1578 for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
1579 snprintf(buf, sizeof(buf), "peer_mw_trans%d", widx);
1580 debugfs_create_file(buf, 0600,
1581 tc->peers[pidx].dbgfs_dir,
1582 &tc->peers[pidx].outmws[widx],
1583 &tool_peer_mw_trans_fops);
1584 }
1585
1586 for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1587 snprintf(buf, sizeof(buf), "spad%d", sidx);
1588
1589 debugfs_create_file(buf, 0600,
1590 tc->peers[pidx].dbgfs_dir,
1591 &tc->peers[pidx].outspads[sidx],
1592 &tool_peer_spad_fops);
1593 }
1594
1595 for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1596 snprintf(buf, sizeof(buf), "msg%d", midx);
1597 debugfs_create_file(buf, 0600,
1598 tc->peers[pidx].dbgfs_dir,
1599 &tc->peers[pidx].outmsgs[midx],
1600 &tool_outmsg_fops);
1601 }
1602 }
1603}
1604
1605static void tool_clear_dbgfs(struct tool_ctx *tc)
1606{
1607 debugfs_remove_recursive(tc->dbgfs_dir);
1608}
1609
1610static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
1611{
1612 struct tool_ctx *tc;
1613 int ret;
1614
1615 tc = tool_create_data(ntb);
1616 if (IS_ERR(tc))
1617 return PTR_ERR(tc);
1618
1619 ret = tool_init_peers(tc);
1620 if (ret != 0)
1621 goto err_clear_data;
1622
1623 ret = tool_init_mws(tc);
1624 if (ret != 0)
1625 goto err_clear_data;
1626
1627 ret = tool_init_spads(tc);
1628 if (ret != 0)
1629 goto err_clear_mws;
1630
1631 ret = tool_init_msgs(tc);
1632 if (ret != 0)
1633 goto err_clear_mws;
1634
1635 ret = tool_init_ntb(tc);
1636 if (ret != 0)
1637 goto err_clear_mws;
1638
1639 tool_setup_dbgfs(tc);
977 1640
978 return 0; 1641 return 0;
979 1642
980err_ctx: 1643err_clear_mws:
981 tool_free_mws(tc); 1644 tool_clear_mws(tc);
982 debugfs_remove_recursive(tc->dbgfs); 1645
983 kfree(tc); 1646err_clear_data:
984err_tc: 1647 tool_clear_data(tc);
985 return rc; 1648
1649 return ret;
986} 1650}
987 1651
988static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb) 1652static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb)
989{ 1653{
990 struct tool_ctx *tc = ntb->ctx; 1654 struct tool_ctx *tc = ntb->ctx;
991 1655
992 tool_free_mws(tc); 1656 tool_clear_dbgfs(tc);
993 1657
994 ntb_clear_ctx(ntb); 1658 tool_clear_ntb(tc);
995 ntb_link_disable(ntb);
996 1659
997 debugfs_remove_recursive(tc->dbgfs); 1660 tool_clear_mws(tc);
998 kfree(tc); 1661
1662 tool_clear_data(tc);
999} 1663}
1000 1664
1001static struct ntb_client tool_client = { 1665static struct ntb_client tool_client = {
1002 .ops = { 1666 .ops = {
1003 .probe = tool_probe, 1667 .probe = tool_probe,
1004 .remove = tool_remove, 1668 .remove = tool_remove,
1005 }, 1669 }
1006}; 1670};
1007 1671
1008static int __init tool_init(void) 1672static int __init tool_init(void)
1009{ 1673{
1010 int rc; 1674 int ret;
1011 1675
1012 if (debugfs_initialized()) 1676 if (debugfs_initialized())
1013 tool_dbgfs = debugfs_create_dir(KBUILD_MODNAME, NULL); 1677 tool_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1014 1678
1015 rc = ntb_register_client(&tool_client); 1679 ret = ntb_register_client(&tool_client);
1016 if (rc) 1680 if (ret)
1017 goto err_client; 1681 debugfs_remove_recursive(tool_dbgfs_topdir);
1018 1682
1019 return 0; 1683 return ret;
1020
1021err_client:
1022 debugfs_remove_recursive(tool_dbgfs);
1023 return rc;
1024} 1684}
1025module_init(tool_init); 1685module_init(tool_init);
1026 1686
1027static void __exit tool_exit(void) 1687static void __exit tool_exit(void)
1028{ 1688{
1029 ntb_unregister_client(&tool_client); 1689 ntb_unregister_client(&tool_client);
1030 debugfs_remove_recursive(tool_dbgfs); 1690 debugfs_remove_recursive(tool_dbgfs_topdir);
1031} 1691}
1032module_exit(tool_exit); 1692module_exit(tool_exit);
1693
diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index c308964777eb..181d16601dd9 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -71,6 +71,7 @@ struct pci_dev;
71 * @NTB_TOPO_B2B_USD: On primary side of local ntb upstream of remote ntb. 71 * @NTB_TOPO_B2B_USD: On primary side of local ntb upstream of remote ntb.
72 * @NTB_TOPO_B2B_DSD: On primary side of local ntb downstream of remote ntb. 72 * @NTB_TOPO_B2B_DSD: On primary side of local ntb downstream of remote ntb.
73 * @NTB_TOPO_SWITCH: Connected via a switch which supports ntb. 73 * @NTB_TOPO_SWITCH: Connected via a switch which supports ntb.
74 * @NTB_TOPO_CROSSLINK: Connected via two symmetric switchecs
74 */ 75 */
75enum ntb_topo { 76enum ntb_topo {
76 NTB_TOPO_NONE = -1, 77 NTB_TOPO_NONE = -1,
@@ -79,6 +80,7 @@ enum ntb_topo {
79 NTB_TOPO_B2B_USD, 80 NTB_TOPO_B2B_USD,
80 NTB_TOPO_B2B_DSD, 81 NTB_TOPO_B2B_DSD,
81 NTB_TOPO_SWITCH, 82 NTB_TOPO_SWITCH,
83 NTB_TOPO_CROSSLINK,
82}; 84};
83 85
84static inline int ntb_topo_is_b2b(enum ntb_topo topo) 86static inline int ntb_topo_is_b2b(enum ntb_topo topo)
@@ -94,12 +96,13 @@ static inline int ntb_topo_is_b2b(enum ntb_topo topo)
94static inline char *ntb_topo_string(enum ntb_topo topo) 96static inline char *ntb_topo_string(enum ntb_topo topo)
95{ 97{
96 switch (topo) { 98 switch (topo) {
97 case NTB_TOPO_NONE: return "NTB_TOPO_NONE"; 99 case NTB_TOPO_NONE: return "NTB_TOPO_NONE";
98 case NTB_TOPO_PRI: return "NTB_TOPO_PRI"; 100 case NTB_TOPO_PRI: return "NTB_TOPO_PRI";
99 case NTB_TOPO_SEC: return "NTB_TOPO_SEC"; 101 case NTB_TOPO_SEC: return "NTB_TOPO_SEC";
100 case NTB_TOPO_B2B_USD: return "NTB_TOPO_B2B_USD"; 102 case NTB_TOPO_B2B_USD: return "NTB_TOPO_B2B_USD";
101 case NTB_TOPO_B2B_DSD: return "NTB_TOPO_B2B_DSD"; 103 case NTB_TOPO_B2B_DSD: return "NTB_TOPO_B2B_DSD";
102 case NTB_TOPO_SWITCH: return "NTB_TOPO_SWITCH"; 104 case NTB_TOPO_SWITCH: return "NTB_TOPO_SWITCH";
105 case NTB_TOPO_CROSSLINK: return "NTB_TOPO_CROSSLINK";
103 } 106 }
104 return "NTB_TOPO_INVALID"; 107 return "NTB_TOPO_INVALID";
105} 108}
@@ -250,7 +253,7 @@ static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops)
250 * @msg_set_mask: See ntb_msg_set_mask(). 253 * @msg_set_mask: See ntb_msg_set_mask().
251 * @msg_clear_mask: See ntb_msg_clear_mask(). 254 * @msg_clear_mask: See ntb_msg_clear_mask().
252 * @msg_read: See ntb_msg_read(). 255 * @msg_read: See ntb_msg_read().
253 * @msg_write: See ntb_msg_write(). 256 * @peer_msg_write: See ntb_peer_msg_write().
254 */ 257 */
255struct ntb_dev_ops { 258struct ntb_dev_ops {
256 int (*port_number)(struct ntb_dev *ntb); 259 int (*port_number)(struct ntb_dev *ntb);
@@ -321,8 +324,8 @@ struct ntb_dev_ops {
321 int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits); 324 int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits);
322 int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits); 325 int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits);
323 int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits); 326 int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits);
324 int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg); 327 u32 (*msg_read)(struct ntb_dev *ntb, int *pidx, int midx);
325 int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg); 328 int (*peer_msg_write)(struct ntb_dev *ntb, int pidx, int midx, u32 msg);
326}; 329};
327 330
328static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops) 331static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
@@ -384,7 +387,7 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
384 /* !ops->msg_set_mask == !ops->msg_count && */ 387 /* !ops->msg_set_mask == !ops->msg_count && */
385 /* !ops->msg_clear_mask == !ops->msg_count && */ 388 /* !ops->msg_clear_mask == !ops->msg_count && */
386 !ops->msg_read == !ops->msg_count && 389 !ops->msg_read == !ops->msg_count &&
387 !ops->msg_write == !ops->msg_count && 390 !ops->peer_msg_write == !ops->msg_count &&
388 1; 391 1;
389} 392}
390 393
@@ -764,7 +767,7 @@ static inline int ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx,
764 resource_size_t *size_align, 767 resource_size_t *size_align,
765 resource_size_t *size_max) 768 resource_size_t *size_max)
766{ 769{
767 if (!(ntb_link_is_up(ntb, NULL, NULL) & (1 << pidx))) 770 if (!(ntb_link_is_up(ntb, NULL, NULL) & BIT_ULL(pidx)))
768 return -ENOTCONN; 771 return -ENOTCONN;
769 772
770 return ntb->ops->mw_get_align(ntb, pidx, widx, addr_align, size_align, 773 return ntb->ops->mw_get_align(ntb, pidx, widx, addr_align, size_align,
@@ -1459,31 +1462,29 @@ static inline int ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits)
1459} 1462}
1460 1463
1461/** 1464/**
1462 * ntb_msg_read() - read message register with specified index 1465 * ntb_msg_read() - read inbound message register with specified index
1463 * @ntb: NTB device context. 1466 * @ntb: NTB device context.
1464 * @midx: Message register index
1465 * @pidx: OUT - Port index of peer device a message retrieved from 1467 * @pidx: OUT - Port index of peer device a message retrieved from
1466 * @msg: OUT - Data 1468 * @midx: Message register index
1467 * 1469 *
1468 * Read data from the specified message register. Source port index of a 1470 * Read data from the specified message register. Source port index of a
1469 * message is retrieved as well. 1471 * message is retrieved as well.
1470 * 1472 *
1471 * Return: Zero on success, otherwise a negative error number. 1473 * Return: The value of the inbound message register.
1472 */ 1474 */
1473static inline int ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, 1475static inline u32 ntb_msg_read(struct ntb_dev *ntb, int *pidx, int midx)
1474 u32 *msg)
1475{ 1476{
1476 if (!ntb->ops->msg_read) 1477 if (!ntb->ops->msg_read)
1477 return -EINVAL; 1478 return ~(u32)0;
1478 1479
1479 return ntb->ops->msg_read(ntb, midx, pidx, msg); 1480 return ntb->ops->msg_read(ntb, pidx, midx);
1480} 1481}
1481 1482
1482/** 1483/**
1483 * ntb_msg_write() - write data to the specified message register 1484 * ntb_peer_msg_write() - write data to the specified peer message register
1484 * @ntb: NTB device context. 1485 * @ntb: NTB device context.
1485 * @midx: Message register index
1486 * @pidx: Port index of peer device a message being sent to 1486 * @pidx: Port index of peer device a message being sent to
1487 * @midx: Message register index
1487 * @msg: Data to send 1488 * @msg: Data to send
1488 * 1489 *
1489 * Send data to a specified peer device using the defined message register. 1490 * Send data to a specified peer device using the defined message register.
@@ -1492,13 +1493,13 @@ static inline int ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx,
1492 * 1493 *
1493 * Return: Zero on success, otherwise a negative error number. 1494 * Return: Zero on success, otherwise a negative error number.
1494 */ 1495 */
1495static inline int ntb_msg_write(struct ntb_dev *ntb, int midx, int pidx, 1496static inline int ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx,
1496 u32 msg) 1497 u32 msg)
1497{ 1498{
1498 if (!ntb->ops->msg_write) 1499 if (!ntb->ops->peer_msg_write)
1499 return -EINVAL; 1500 return -EINVAL;
1500 1501
1501 return ntb->ops->msg_write(ntb, midx, pidx, msg); 1502 return ntb->ops->peer_msg_write(ntb, pidx, midx, msg);
1502} 1503}
1503 1504
1504#endif 1505#endif
diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h
index 09d73d0d1aa8..6d325a7a0c19 100644
--- a/include/linux/switchtec.h
+++ b/include/linux/switchtec.h
@@ -168,6 +168,14 @@ struct ntb_info_regs {
168 u16 reserved1; 168 u16 reserved1;
169 u64 ep_map; 169 u64 ep_map;
170 u16 requester_id; 170 u16 requester_id;
171 u16 reserved2;
172 u32 reserved3[4];
173 struct nt_partition_info {
174 u32 xlink_enabled;
175 u32 target_part_low;
176 u32 target_part_high;
177 u32 reserved;
178 } ntp_info[48];
171} __packed; 179} __packed;
172 180
173struct part_cfg_regs { 181struct part_cfg_regs {
@@ -284,7 +292,20 @@ enum {
284struct pff_csr_regs { 292struct pff_csr_regs {
285 u16 vendor_id; 293 u16 vendor_id;
286 u16 device_id; 294 u16 device_id;
287 u32 pci_cfg_header[15]; 295 u16 pcicmd;
296 u16 pcists;
297 u32 pci_class;
298 u32 pci_opts;
299 union {
300 u32 pci_bar[6];
301 u64 pci_bar64[3];
302 };
303 u32 pci_cardbus;
304 u32 pci_subsystem_id;
305 u32 pci_expansion_rom;
306 u32 pci_cap_ptr;
307 u32 reserved1;
308 u32 pci_irq;
288 u32 pci_cap_region[48]; 309 u32 pci_cap_region[48];
289 u32 pcie_cap_region[448]; 310 u32 pcie_cap_region[448];
290 u32 indirect_gas_window[128]; 311 u32 indirect_gas_window[128];
diff --git a/tools/testing/selftests/ntb/ntb_test.sh b/tools/testing/selftests/ntb/ntb_test.sh
index 5fc7ad359e21..08cbfbbc7029 100755
--- a/tools/testing/selftests/ntb/ntb_test.sh
+++ b/tools/testing/selftests/ntb/ntb_test.sh
@@ -18,7 +18,6 @@ LIST_DEVS=FALSE
18 18
19DEBUGFS=${DEBUGFS-/sys/kernel/debug} 19DEBUGFS=${DEBUGFS-/sys/kernel/debug}
20 20
21DB_BITMASK=0x7FFF
22PERF_RUN_ORDER=32 21PERF_RUN_ORDER=32
23MAX_MW_SIZE=0 22MAX_MW_SIZE=0
24RUN_DMA_TESTS= 23RUN_DMA_TESTS=
@@ -39,15 +38,17 @@ function show_help()
39 echo "be highly recommended." 38 echo "be highly recommended."
40 echo 39 echo
41 echo "Options:" 40 echo "Options:"
42 echo " -b BITMASK doorbell clear bitmask for ntb_tool"
43 echo " -C don't cleanup ntb modules on exit" 41 echo " -C don't cleanup ntb modules on exit"
44 echo " -d run dma tests"
45 echo " -h show this help message" 42 echo " -h show this help message"
46 echo " -l list available local and remote PCI ids" 43 echo " -l list available local and remote PCI ids"
47 echo " -r REMOTE_HOST specify the remote's hostname to connect" 44 echo " -r REMOTE_HOST specify the remote's hostname to connect"
48 echo " to for the test (using ssh)" 45 echo " to for the test (using ssh)"
49 echo " -p NUM ntb_perf run order (default: $PERF_RUN_ORDER)" 46 echo " -m MW_SIZE memory window size for ntb_tool"
50 echo " -w max_mw_size maxmium memory window size" 47 echo " (default: $MW_SIZE)"
48 echo " -d run dma tests for ntb_perf"
49 echo " -p ORDER total data order for ntb_perf"
50 echo " (default: $PERF_RUN_ORDER)"
51 echo " -w MAX_MW_SIZE maxmium memory window size for ntb_perf"
51 echo 52 echo
52} 53}
53 54
@@ -56,7 +57,6 @@ function parse_args()
56 OPTIND=0 57 OPTIND=0
57 while getopts "b:Cdhlm:r:p:w:" opt; do 58 while getopts "b:Cdhlm:r:p:w:" opt; do
58 case "$opt" in 59 case "$opt" in
59 b) DB_BITMASK=${OPTARG} ;;
60 C) DONT_CLEANUP=1 ;; 60 C) DONT_CLEANUP=1 ;;
61 d) RUN_DMA_TESTS=1 ;; 61 d) RUN_DMA_TESTS=1 ;;
62 h) show_help; exit 0 ;; 62 h) show_help; exit 0 ;;
@@ -87,7 +87,7 @@ set -e
87 87
88function _modprobe() 88function _modprobe()
89{ 89{
90 modprobe "$@" 90 modprobe "$@"
91 91
92 if [[ "$REMOTE_HOST" != "" ]]; then 92 if [[ "$REMOTE_HOST" != "" ]]; then
93 ssh "$REMOTE_HOST" modprobe "$@" 93 ssh "$REMOTE_HOST" modprobe "$@"
@@ -127,15 +127,70 @@ function write_file()
127 fi 127 fi
128} 128}
129 129
130function check_file()
131{
132 split_remote $1
133
134 if [[ "$REMOTE" != "" ]]; then
135 ssh "$REMOTE" "[[ -e ${VPATH} ]]"
136 else
137 [[ -e ${VPATH} ]]
138 fi
139}
140
141function subdirname()
142{
143 echo $(basename $(dirname $1)) 2> /dev/null
144}
145
146function find_pidx()
147{
148 PORT=$1
149 PPATH=$2
150
151 for ((i = 0; i < 64; i++)); do
152 PEER_DIR="$PPATH/peer$i"
153
154 check_file ${PEER_DIR} || break
155
156 PEER_PORT=$(read_file "${PEER_DIR}/port")
157 if [[ ${PORT} -eq $PEER_PORT ]]; then
158 echo $i
159 return 0
160 fi
161 done
162
163 return 1
164}
165
166function port_test()
167{
168 LOC=$1
169 REM=$2
170
171 echo "Running port tests on: $(basename $LOC) / $(basename $REM)"
172
173 LOCAL_PORT=$(read_file "$LOC/port")
174 REMOTE_PORT=$(read_file "$REM/port")
175
176 LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")
177 REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")
178
179 echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"
180 echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"
181
182 echo " Passed"
183}
184
130function link_test() 185function link_test()
131{ 186{
132 LOC=$1 187 LOC=$1
133 REM=$2 188 REM=$2
134 EXP=0 189 EXP=0
135 190
136 echo "Running link tests on: $(basename $LOC) / $(basename $REM)" 191 echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"
137 192
138 if ! write_file "N" "$LOC/link" 2> /dev/null; then 193 if ! write_file "N" "$LOC/../link" 2> /dev/null; then
139 echo " Unsupported" 194 echo " Unsupported"
140 return 195 return
141 fi 196 fi
@@ -143,12 +198,11 @@ function link_test()
143 write_file "N" "$LOC/link_event" 198 write_file "N" "$LOC/link_event"
144 199
145 if [[ $(read_file "$REM/link") != "N" ]]; then 200 if [[ $(read_file "$REM/link") != "N" ]]; then
146 echo "Expected remote link to be down in $REM/link" >&2 201 echo "Expected link to be down in $REM/link" >&2
147 exit -1 202 exit -1
148 fi 203 fi
149 204
150 write_file "Y" "$LOC/link" 205 write_file "Y" "$LOC/../link"
151 write_file "Y" "$LOC/link_event"
152 206
153 echo " Passed" 207 echo " Passed"
154} 208}
@@ -161,58 +215,136 @@ function doorbell_test()
161 215
162 echo "Running db tests on: $(basename $LOC) / $(basename $REM)" 216 echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
163 217
164 write_file "c $DB_BITMASK" "$REM/db" 218 DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")
219
220 write_file "c $DB_VALID_MASK" "$REM/db"
165 221
166 for ((i=1; i <= 8; i++)); do 222 for ((i = 0; i < 64; i++)); do
167 let DB=$(read_file "$REM/db") || true 223 DB=$(read_file "$REM/db")
168 if [[ "$DB" != "$EXP" ]]; then 224 if [[ "$DB" -ne "$EXP" ]]; then
169 echo "Doorbell doesn't match expected value $EXP " \ 225 echo "Doorbell doesn't match expected value $EXP " \
170 "in $REM/db" >&2 226 "in $REM/db" >&2
171 exit -1 227 exit -1
172 fi 228 fi
173 229
174 let "MASK=1 << ($i-1)" || true 230 let "MASK = (1 << $i) & $DB_VALID_MASK" || true
175 let "EXP=$EXP | $MASK" || true 231 let "EXP = $EXP | $MASK" || true
232
176 write_file "s $MASK" "$LOC/peer_db" 233 write_file "s $MASK" "$LOC/peer_db"
177 done 234 done
178 235
236 write_file "c $DB_VALID_MASK" "$REM/db_mask"
237 write_file $DB_VALID_MASK "$REM/db_event"
238 write_file "s $DB_VALID_MASK" "$REM/db_mask"
239
240 write_file "c $DB_VALID_MASK" "$REM/db"
241
179 echo " Passed" 242 echo " Passed"
180} 243}
181 244
182function read_spad() 245function get_files_count()
183{ 246{
184 VPATH=$1 247 NAME=$1
185 IDX=$2 248 LOC=$2
249
250 split_remote $LOC
186 251
187 ROW=($(read_file "$VPATH" | grep -e "^$IDX")) 252 if [[ "$REMOTE" == "" ]]; then
188 let VAL=${ROW[1]} || true 253 echo $(ls -1 "$LOC"/${NAME}* 2>/dev/null | wc -l)
189 echo $VAL 254 else
255 echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \
256 wc -l" 2> /dev/null)
257 fi
190} 258}
191 259
192function scratchpad_test() 260function scratchpad_test()
193{ 261{
194 LOC=$1 262 LOC=$1
195 REM=$2 263 REM=$2
196 CNT=$(read_file "$LOC/spad" | wc -l)
197 264
198 echo "Running spad tests on: $(basename $LOC) / $(basename $REM)" 265 echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"
266
267 CNT=$(get_files_count "spad" "$LOC")
268
269 if [[ $CNT -eq 0 ]]; then
270 echo " Unsupported"
271 return
272 fi
199 273
200 for ((i = 0; i < $CNT; i++)); do 274 for ((i = 0; i < $CNT; i++)); do
201 VAL=$RANDOM 275 VAL=$RANDOM
202 write_file "$i $VAL" "$LOC/peer_spad" 276 write_file "$VAL" "$LOC/spad$i"
203 RVAL=$(read_spad "$REM/spad" $i) 277 RVAL=$(read_file "$REM/../spad$i")
204 278
205 if [[ "$VAL" != "$RVAL" ]]; then 279 if [[ "$VAL" -ne "$RVAL" ]]; then
206 echo "Scratchpad doesn't match expected value $VAL " \ 280 echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2
207 "in $REM/spad, got $RVAL" >&2
208 exit -1 281 exit -1
209 fi 282 fi
283 done
284
285 echo " Passed"
286}
287
288function message_test()
289{
290 LOC=$1
291 REM=$2
292
293 echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"
294
295 CNT=$(get_files_count "msg" "$LOC")
210 296
297 if [[ $CNT -eq 0 ]]; then
298 echo " Unsupported"
299 return
300 fi
301
302 MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")
303 MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")
304
305 write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"
306 write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"
307
308 for ((i = 0; i < $CNT; i++)); do
309 VAL=$RANDOM
310 write_file "$VAL" "$LOC/msg$i"
311 RVAL=$(read_file "$REM/../msg$i")
312
313 if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then
314 echo "Message $i value $RVAL doesn't match $VAL" >&2
315 exit -1
316 fi
211 done 317 done
212 318
213 echo " Passed" 319 echo " Passed"
214} 320}
215 321
322function get_number()
323{
324 KEY=$1
325
326 sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"
327}
328
329function mw_alloc()
330{
331 IDX=$1
332 LOC=$2
333 REM=$3
334
335 write_file $MW_SIZE "$LOC/mw_trans$IDX"
336
337 INB_MW=$(read_file "$LOC/mw_trans$IDX")
338 MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")
339 MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")
340
341 write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"
342
343 if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then
344 echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"
345 fi
346}
347
216function write_mw() 348function write_mw()
217{ 349{
218 split_remote $2 350 split_remote $2
@@ -225,17 +357,15 @@ function write_mw()
225 fi 357 fi
226} 358}
227 359
228function mw_test() 360function mw_check()
229{ 361{
230 IDX=$1 362 IDX=$1
231 LOC=$2 363 LOC=$2
232 REM=$3 364 REM=$3
233 365
234 echo "Running $IDX tests on: $(basename $LOC) / $(basename $REM)" 366 write_mw "$LOC/mw$IDX"
235 367
236 write_mw "$LOC/$IDX" 368 split_remote "$LOC/mw$IDX"
237
238 split_remote "$LOC/$IDX"
239 if [[ "$REMOTE" == "" ]]; then 369 if [[ "$REMOTE" == "" ]]; then
240 A=$VPATH 370 A=$VPATH
241 else 371 else
@@ -243,7 +373,7 @@ function mw_test()
243 ssh "$REMOTE" cat "$VPATH" > "$A" 373 ssh "$REMOTE" cat "$VPATH" > "$A"
244 fi 374 fi
245 375
246 split_remote "$REM/peer_$IDX" 376 split_remote "$REM/peer_mw$IDX"
247 if [[ "$REMOTE" == "" ]]; then 377 if [[ "$REMOTE" == "" ]]; then
248 B=$VPATH 378 B=$VPATH
249 else 379 else
@@ -251,7 +381,7 @@ function mw_test()
251 ssh "$REMOTE" cat "$VPATH" > "$B" 381 ssh "$REMOTE" cat "$VPATH" > "$B"
252 fi 382 fi
253 383
254 cmp -n $MW_SIZE "$A" "$B" 384 cmp -n $MW_ALIGNED_SIZE "$A" "$B"
255 if [[ $? != 0 ]]; then 385 if [[ $? != 0 ]]; then
256 echo "Memory window $MW did not match!" >&2 386 echo "Memory window $MW did not match!" >&2
257 fi 387 fi
@@ -263,8 +393,39 @@ function mw_test()
263 if [[ "$B" == "/tmp/*" ]]; then 393 if [[ "$B" == "/tmp/*" ]]; then
264 rm "$B" 394 rm "$B"
265 fi 395 fi
396}
397
398function mw_free()
399{
400 IDX=$1
401 LOC=$2
402 REM=$3
403
404 write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"
405
406 write_file 0 "$LOC/mw_trans$IDX"
407}
408
409function mw_test()
410{
411 LOC=$1
412 REM=$2
413
414 CNT=$(get_files_count "mw_trans" "$LOC")
415
416 for ((i = 0; i < $CNT; i++)); do
417 echo "Running mw$i tests on: $(subdirname $LOC) / " \
418 "$(subdirname $REM)"
419
420 mw_alloc $i $LOC $REM
421
422 mw_check $i $LOC $REM
423
424 mw_free $i $LOC $REM
425
426 echo " Passed"
427 done
266 428
267 echo " Passed"
268} 429}
269 430
270function pingpong_test() 431function pingpong_test()
@@ -274,13 +435,13 @@ function pingpong_test()
274 435
275 echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)" 436 echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
276 437
277 LOC_START=$(read_file $LOC/count) 438 LOC_START=$(read_file "$LOC/count")
278 REM_START=$(read_file $REM/count) 439 REM_START=$(read_file "$REM/count")
279 440
280 sleep 7 441 sleep 7
281 442
282 LOC_END=$(read_file $LOC/count) 443 LOC_END=$(read_file "$LOC/count")
283 REM_END=$(read_file $REM/count) 444 REM_END=$(read_file "$REM/count")
284 445
285 if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then 446 if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
286 echo "Ping pong counter not incrementing!" >&2 447 echo "Ping pong counter not incrementing!" >&2
@@ -300,19 +461,19 @@ function perf_test()
300 WITH="without" 461 WITH="without"
301 fi 462 fi
302 463
303 _modprobe ntb_perf run_order=$PERF_RUN_ORDER \ 464 _modprobe ntb_perf total_order=$PERF_RUN_ORDER \
304 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA 465 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
305 466
306 echo "Running local perf test $WITH DMA" 467 echo "Running local perf test $WITH DMA"
307 write_file "" $LOCAL_PERF/run 468 write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"
308 echo -n " " 469 echo -n " "
309 read_file $LOCAL_PERF/run 470 read_file "$LOCAL_PERF/run"
310 echo " Passed" 471 echo " Passed"
311 472
312 echo "Running remote perf test $WITH DMA" 473 echo "Running remote perf test $WITH DMA"
313 write_file "" $REMOTE_PERF/run 474 write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"
314 echo -n " " 475 echo -n " "
315 read_file $REMOTE_PERF/run 476 read_file "$REMOTE_PERF/run"
316 echo " Passed" 477 echo " Passed"
317 478
318 _modprobe -r ntb_perf 479 _modprobe -r ntb_perf
@@ -320,48 +481,44 @@ function perf_test()
320 481
321function ntb_tool_tests() 482function ntb_tool_tests()
322{ 483{
323 LOCAL_TOOL=$DEBUGFS/ntb_tool/$LOCAL_DEV 484 LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"
324 REMOTE_TOOL=$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV 485 REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"
325 486
326 echo "Starting ntb_tool tests..." 487 echo "Starting ntb_tool tests..."
327 488
328 _modprobe ntb_tool 489 _modprobe ntb_tool
329 490
330 write_file Y $LOCAL_TOOL/link_event 491 port_test "$LOCAL_TOOL" "$REMOTE_TOOL"
331 write_file Y $REMOTE_TOOL/link_event
332 492
333 link_test $LOCAL_TOOL $REMOTE_TOOL 493 LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"
334 link_test $REMOTE_TOOL $LOCAL_TOOL 494 REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"
495
496 link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
497 link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
335 498
336 #Ensure the link is up on both sides before continuing 499 #Ensure the link is up on both sides before continuing
337 write_file Y $LOCAL_TOOL/link_event 500 write_file "Y" "$LOCAL_PEER_TOOL/link_event"
338 write_file Y $REMOTE_TOOL/link_event 501 write_file "Y" "$REMOTE_PEER_TOOL/link_event"
339 502
340 for PEER_TRANS in $(ls $LOCAL_TOOL/peer_trans*); do 503 doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"
341 PT=$(basename $PEER_TRANS) 504 doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"
342 write_file $MW_SIZE $LOCAL_TOOL/$PT
343 write_file $MW_SIZE $REMOTE_TOOL/$PT
344 done
345 505
346 doorbell_test $LOCAL_TOOL $REMOTE_TOOL 506 scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
347 doorbell_test $REMOTE_TOOL $LOCAL_TOOL 507 scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
348 scratchpad_test $LOCAL_TOOL $REMOTE_TOOL
349 scratchpad_test $REMOTE_TOOL $LOCAL_TOOL
350 508
351 for MW in $(ls $LOCAL_TOOL/mw*); do 509 message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
352 MW=$(basename $MW) 510 message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
353 511
354 mw_test $MW $LOCAL_TOOL $REMOTE_TOOL 512 mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
355 mw_test $MW $REMOTE_TOOL $LOCAL_TOOL 513 mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
356 done
357 514
358 _modprobe -r ntb_tool 515 _modprobe -r ntb_tool
359} 516}
360 517
361function ntb_pingpong_tests() 518function ntb_pingpong_tests()
362{ 519{
363 LOCAL_PP=$DEBUGFS/ntb_pingpong/$LOCAL_DEV 520 LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"
364 REMOTE_PP=$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV 521 REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"
365 522
366 echo "Starting ntb_pingpong tests..." 523 echo "Starting ntb_pingpong tests..."
367 524
@@ -374,8 +531,8 @@ function ntb_pingpong_tests()
374 531
375function ntb_perf_tests() 532function ntb_perf_tests()
376{ 533{
377 LOCAL_PERF=$DEBUGFS/ntb_perf/$LOCAL_DEV 534 LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"
378 REMOTE_PERF=$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV 535 REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"
379 536
380 echo "Starting ntb_perf tests..." 537 echo "Starting ntb_perf tests..."
381 538
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-22 02:53:11 -0400