aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAllen Hubbe <Allen.Hubbe@emc.com>2015-04-09 10:33:20 -0400
committerJon Mason <jdmason@kudzu.us>2015-07-04 14:05:49 -0400
commite26a5843f7f5014ae4460030ca4de029a3ac35d3 (patch)
treea16ffee275c8e43e18bfd3424415e3a36c4f3c7a
parenta1bd3baeb2f18b2b3d0f98ce5fdaa725149b950b (diff)
NTB: Split ntb_hw_intel and ntb_transport drivers
Change ntb_hw_intel to use the new NTB hardware abstraction layer. Split ntb_transport into its own driver. Change it to use the new NTB hardware abstraction layer. Signed-off-by: Allen Hubbe <Allen.Hubbe@emc.com> Signed-off-by: Jon Mason <jdmason@kudzu.us>
Diffstat
-rw-r--r--Documentation/ntb.txt26
-rw-r--r--MAINTAINERS8
-rw-r--r--drivers/net/ntb_netdev.c54
-rw-r--r--drivers/ntb/Kconfig37
-rw-r--r--drivers/ntb/Makefile6
-rw-r--r--drivers/ntb/hw/Kconfig1
-rw-r--r--drivers/ntb/hw/Makefile1
-rw-r--r--drivers/ntb/hw/intel/Kconfig7
-rw-r--r--drivers/ntb/hw/intel/Makefile1
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_intel.c3073
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_intel.h607
-rw-r--r--drivers/ntb/ntb_transport.c936
-rw-r--r--include/linux/ntb_transport.h25
13 files changed, 2588 insertions, 2194 deletions
diff --git a/Documentation/ntb.txt b/Documentation/ntb.txt
index 9d46dc9712a8..725ba1e6c127 100644
--- a/Documentation/ntb.txt
+++ b/Documentation/ntb.txt
@@ -26,7 +26,33 @@ as ntb hardware, or hardware drivers, are inserted and removed. The
26registration uses the Linux Device framework, so it should feel familiar to 26registration uses the Linux Device framework, so it should feel familiar to
27anyone who has written a pci driver. 27anyone who has written a pci driver.
28 28
29### NTB Transport Client (ntb\_transport) and NTB Netdev (ntb\_netdev)
30
31The primary client for NTB is the Transport client, used in tandem with NTB
32Netdev. These drivers function together to create a logical link to the peer,
33across the ntb, to exchange packets of network data. The Transport client
34establishes a logical link to the peer, and creates queue pairs to exchange
35messages and data. The NTB Netdev then creates an ethernet device using a
36Transport queue pair. Network data is copied between socket buffers and the
37Transport queue pair buffer. The Transport client may be used for other things
38besides Netdev, however no other applications have yet been written.
39
29## NTB Hardware Drivers 40## NTB Hardware Drivers
30 41
31NTB hardware drivers should register devices with the NTB core driver. After 42NTB hardware drivers should register devices with the NTB core driver. After
32registering, clients probe and remove functions will be called. 43registering, clients probe and remove functions will be called.
44
45### NTB Intel Hardware Driver (ntb\_hw\_intel)
46
47The Intel hardware driver supports NTB on Xeon and Atom CPUs.
48
49Module Parameters:
50
51* b2b\_mw\_idx - If the peer ntb is to be accessed via a memory window, then use
52 this memory window to access the peer ntb. A value of zero or positive
53 starts from the first mw idx, and a negative value starts from the last
54 mw idx. Both sides MUST set the same value here! The default value is
55 `-1`.
56* b2b\_mw\_share - If the peer ntb is to be accessed via a memory window, and if
57 the memory window is large enough, still allow the client to use the
58 second half of the memory window for address translation to the peer.
diff --git a/MAINTAINERS b/MAINTAINERS
index e2fc9eec2e16..a682805d6d56 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -7007,6 +7007,14 @@ F: drivers/net/ntb_netdev.c
7007F: include/linux/ntb.h 7007F: include/linux/ntb.h
7008F: include/linux/ntb_transport.h 7008F: include/linux/ntb_transport.h
7009 7009
7010NTB INTEL DRIVER
7011M: Jon Mason <jdmason@kudzu.us>
7012M: Dave Jiang <dave.jiang@intel.com>
7013S: Supported
7014W: https://github.com/jonmason/ntb/wiki
7015T: git git://github.com/jonmason/ntb.git
7016F: drivers/ntb/hw/intel/
7017
7010NTFS FILESYSTEM 7018NTFS FILESYSTEM
7011M: Anton Altaparmakov <anton@tuxera.com> 7019M: Anton Altaparmakov <anton@tuxera.com>
7012L: linux-ntfs-dev@lists.sourceforge.net 7020L: linux-ntfs-dev@lists.sourceforge.net
diff --git a/drivers/net/ntb_netdev.c b/drivers/net/ntb_netdev.c
index 6d3bfa62f5ec..3cc316cb7e6b 100644
--- a/drivers/net/ntb_netdev.c
+++ b/drivers/net/ntb_netdev.c
@@ -5,6 +5,7 @@
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
13 * BSD LICENSE 14 * BSD LICENSE
14 * 15 *
15 * Copyright(c) 2012 Intel Corporation. All rights reserved. 16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
16 * 18 *
17 * Redistribution and use in source and binary forms, with or without 19 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions 20 * modification, are permitted provided that the following conditions
@@ -40,7 +42,7 @@
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 * 44 *
43 * Intel PCIe NTB Network Linux driver 45 * PCIe NTB Network Linux driver
44 * 46 *
45 * Contact Information: 47 * Contact Information:
46 * Jon Mason <jon.mason@intel.com> 48 * Jon Mason <jon.mason@intel.com>
@@ -49,6 +51,7 @@
49#include <linux/ethtool.h> 51#include <linux/ethtool.h>
50#include <linux/module.h> 52#include <linux/module.h>
51#include <linux/pci.h> 53#include <linux/pci.h>
54#include <linux/ntb.h>
52#include <linux/ntb_transport.h> 55#include <linux/ntb_transport.h>
53 56
54#define NTB_NETDEV_VER "0.7" 57#define NTB_NETDEV_VER "0.7"
@@ -70,26 +73,19 @@ struct ntb_netdev {
70 73
71static LIST_HEAD(dev_list); 74static LIST_HEAD(dev_list);
72 75
73static void ntb_netdev_event_handler(void *data, int status) 76static void ntb_netdev_event_handler(void *data, int link_is_up)
74{ 77{
75 struct net_device *ndev = data; 78 struct net_device *ndev = data;
76 struct ntb_netdev *dev = netdev_priv(ndev); 79 struct ntb_netdev *dev = netdev_priv(ndev);
77 80
78 netdev_dbg(ndev, "Event %x, Link %x\n", status, 81 netdev_dbg(ndev, "Event %x, Link %x\n", link_is_up,
79 ntb_transport_link_query(dev->qp)); 82 ntb_transport_link_query(dev->qp));
80 83
81 switch (status) { 84 if (link_is_up) {
82 case NTB_LINK_DOWN: 85 if (ntb_transport_link_query(dev->qp))
86 netif_carrier_on(ndev);
87 } else {
83 netif_carrier_off(ndev); 88 netif_carrier_off(ndev);
84 break;
85 case NTB_LINK_UP:
86 if (!ntb_transport_link_query(dev->qp))
87 return;
88
89 netif_carrier_on(ndev);
90 break;
91 default:
92 netdev_warn(ndev, "Unsupported event type %d\n", status);
93 } 89 }
94} 90}
95 91
@@ -160,8 +156,6 @@ static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb,
160 struct ntb_netdev *dev = netdev_priv(ndev); 156 struct ntb_netdev *dev = netdev_priv(ndev);
161 int rc; 157 int rc;
162 158
163 netdev_dbg(ndev, "%s: skb len %d\n", __func__, skb->len);
164
165 rc = ntb_transport_tx_enqueue(dev->qp, skb, skb->data, skb->len); 159 rc = ntb_transport_tx_enqueue(dev->qp, skb, skb->data, skb->len);
166 if (rc) 160 if (rc)
167 goto err; 161 goto err;
@@ -322,20 +316,26 @@ static const struct ntb_queue_handlers ntb_netdev_handlers = {
322 .event_handler = ntb_netdev_event_handler, 316 .event_handler = ntb_netdev_event_handler,
323}; 317};
324 318
325static int ntb_netdev_probe(struct pci_dev *pdev) 319static int ntb_netdev_probe(struct device *client_dev)
326{ 320{
321 struct ntb_dev *ntb;
327 struct net_device *ndev; 322 struct net_device *ndev;
323 struct pci_dev *pdev;
328 struct ntb_netdev *dev; 324 struct ntb_netdev *dev;
329 int rc; 325 int rc;
330 326
331 ndev = alloc_etherdev(sizeof(struct ntb_netdev)); 327 ntb = dev_ntb(client_dev->parent);
328 pdev = ntb->pdev;
329 if (!pdev)
330 return -ENODEV;
331
332 ndev = alloc_etherdev(sizeof(*dev));
332 if (!ndev) 333 if (!ndev)
333 return -ENOMEM; 334 return -ENOMEM;
334 335
335 dev = netdev_priv(ndev); 336 dev = netdev_priv(ndev);
336 dev->ndev = ndev; 337 dev->ndev = ndev;
337 dev->pdev = pdev; 338 dev->pdev = pdev;
338 BUG_ON(!dev->pdev);
339 ndev->features = NETIF_F_HIGHDMA; 339 ndev->features = NETIF_F_HIGHDMA;
340 340
341 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 341 ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
@@ -349,7 +349,8 @@ static int ntb_netdev_probe(struct pci_dev *pdev)
349 ndev->netdev_ops = &ntb_netdev_ops; 349 ndev->netdev_ops = &ntb_netdev_ops;
350 ndev->ethtool_ops = &ntb_ethtool_ops; 350 ndev->ethtool_ops = &ntb_ethtool_ops;
351 351
352 dev->qp = ntb_transport_create_queue(ndev, pdev, &ntb_netdev_handlers); 352 dev->qp = ntb_transport_create_queue(ndev, client_dev,
353 &ntb_netdev_handlers);
353 if (!dev->qp) { 354 if (!dev->qp) {
354 rc = -EIO; 355 rc = -EIO;
355 goto err; 356 goto err;
@@ -372,12 +373,17 @@ err:
372 return rc; 373 return rc;
373} 374}
374 375
375static void ntb_netdev_remove(struct pci_dev *pdev) 376static void ntb_netdev_remove(struct device *client_dev)
376{ 377{
378 struct ntb_dev *ntb;
377 struct net_device *ndev; 379 struct net_device *ndev;
380 struct pci_dev *pdev;
378 struct ntb_netdev *dev; 381 struct ntb_netdev *dev;
379 bool found = false; 382 bool found = false;
380 383
384 ntb = dev_ntb(client_dev->parent);
385 pdev = ntb->pdev;
386
381 list_for_each_entry(dev, &dev_list, list) { 387 list_for_each_entry(dev, &dev_list, list) {
382 if (dev->pdev == pdev) { 388 if (dev->pdev == pdev) {
383 found = true; 389 found = true;
@@ -396,7 +402,7 @@ static void ntb_netdev_remove(struct pci_dev *pdev)
396 free_netdev(ndev); 402 free_netdev(ndev);
397} 403}
398 404
399static struct ntb_client ntb_netdev_client = { 405static struct ntb_transport_client ntb_netdev_client = {
400 .driver.name = KBUILD_MODNAME, 406 .driver.name = KBUILD_MODNAME,
401 .driver.owner = THIS_MODULE, 407 .driver.owner = THIS_MODULE,
402 .probe = ntb_netdev_probe, 408 .probe = ntb_netdev_probe,
@@ -407,7 +413,7 @@ static int __init ntb_netdev_init_module(void)
407{ 413{
408 int rc; 414 int rc;
409 415
410 rc = ntb_register_client_dev(KBUILD_MODNAME); 416 rc = ntb_transport_register_client_dev(KBUILD_MODNAME);
411 if (rc) 417 if (rc)
412 return rc; 418 return rc;
413 return ntb_transport_register_client(&ntb_netdev_client); 419 return ntb_transport_register_client(&ntb_netdev_client);
@@ -417,6 +423,6 @@ module_init(ntb_netdev_init_module);
417static void __exit ntb_netdev_exit_module(void) 423static void __exit ntb_netdev_exit_module(void)
418{ 424{
419 ntb_transport_unregister_client(&ntb_netdev_client); 425 ntb_transport_unregister_client(&ntb_netdev_client);
420 ntb_unregister_client_dev(KBUILD_MODNAME); 426 ntb_transport_unregister_client_dev(KBUILD_MODNAME);
421} 427}
422module_exit(ntb_netdev_exit_module); 428module_exit(ntb_netdev_exit_module);
diff --git a/drivers/ntb/Kconfig b/drivers/ntb/Kconfig
index f69df793dbe2..53b042429673 100644
--- a/drivers/ntb/Kconfig
+++ b/drivers/ntb/Kconfig
@@ -1,13 +1,26 @@
1config NTB 1menuconfig NTB
2 tristate "Intel Non-Transparent Bridge support" 2 tristate "Non-Transparent Bridge support"
3 depends on PCI 3 depends on PCI
4 depends on X86 4 help
5 help 5 The PCI-E Non-transparent bridge hardware is a point-to-point PCI-E bus
6 The PCI-E Non-transparent bridge hardware is a point-to-point PCI-E bus 6 connecting 2 systems. When configured, writes to the device's PCI
7 connecting 2 systems. When configured, writes to the device's PCI 7 mapped memory will be mirrored to a buffer on the remote system. The
8 mapped memory will be mirrored to a buffer on the remote system. The 8 ntb Linux driver uses this point-to-point communication as a method to
9 ntb Linux driver uses this point-to-point communication as a method to 9 transfer data from one system to the other.
10 transfer data from one system to the other.
11
12 If unsure, say N.
13 10
11 If unsure, say N.
12
13if NTB
14
15source "drivers/ntb/hw/Kconfig"
16
17config NTB_TRANSPORT
18 tristate "NTB Transport Client"
19 help
20 This is a transport driver that enables connected systems to exchange
21 messages over the ntb hardware. The transport exposes a queue pair api
22 to client drivers.
23
24 If unsure, say N.
25
26endif # NTB
diff --git a/drivers/ntb/Makefile b/drivers/ntb/Makefile
index 712e953a8fda..b9fa663ecfec 100644
--- a/drivers/ntb/Makefile
+++ b/drivers/ntb/Makefile
@@ -1,4 +1,2 @@
1obj-$(CONFIG_NTB) += ntb.o 1obj-$(CONFIG_NTB) += ntb.o hw/
2obj-$(CONFIG_NTB) += ntb_hw_intel.o 2obj-$(CONFIG_NTB_TRANSPORT) += ntb_transport.o
3
4ntb_hw_intel-objs := hw/intel/ntb_hw_intel.o ntb_transport.o
diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
new file mode 100644
index 000000000000..4d5535c4cddf
--- /dev/null
+++ b/drivers/ntb/hw/Kconfig
@@ -0,0 +1 @@
source "drivers/ntb/hw/intel/Kconfig"
diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
new file mode 100644
index 000000000000..175d7c92a569
--- /dev/null
+++ b/drivers/ntb/hw/Makefile
@@ -0,0 +1 @@
obj-$(CONFIG_NTB_INTEL) += intel/
diff --git a/drivers/ntb/hw/intel/Kconfig b/drivers/ntb/hw/intel/Kconfig
new file mode 100644
index 000000000000..91f995e33ac6
--- /dev/null
+++ b/drivers/ntb/hw/intel/Kconfig
@@ -0,0 +1,7 @@
1config NTB_INTEL
2 tristate "Intel Non-Transparent Bridge support"
3 depends on X86_64
4 help
5 This driver supports Intel NTB on capable Xeon and Atom hardware.
6
7 If unsure, say N.
diff --git a/drivers/ntb/hw/intel/Makefile b/drivers/ntb/hw/intel/Makefile
new file mode 100644
index 000000000000..1b434568d2ad
--- /dev/null
+++ b/drivers/ntb/hw/intel/Makefile
@@ -0,0 +1 @@
obj-$(CONFIG_NTB_INTEL) += ntb_hw_intel.o
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 044534a995f1..686091756ba4 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -5,6 +5,7 @@
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
13 * BSD LICENSE 14 * BSD LICENSE
14 * 15 *
15 * Copyright(c) 2012 Intel Corporation. All rights reserved. 16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
16 * 18 *
17 * Redistribution and use in source and binary forms, with or without 19 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions 20 * modification, are permitted provided that the following conditions
@@ -45,6 +47,7 @@
45 * Contact Information: 47 * Contact Information:
46 * Jon Mason <jon.mason@intel.com> 48 * Jon Mason <jon.mason@intel.com>
47 */ 49 */
50
48#include <linux/debugfs.h> 51#include <linux/debugfs.h>
49#include <linux/delay.h> 52#include <linux/delay.h>
50#include <linux/init.h> 53#include <linux/init.h>
@@ -53,99 +56,97 @@
53#include <linux/pci.h> 56#include <linux/pci.h>
54#include <linux/random.h> 57#include <linux/random.h>
55#include <linux/slab.h> 58#include <linux/slab.h>
59#include <linux/ntb.h>
60
56#include "ntb_hw_intel.h" 61#include "ntb_hw_intel.h"
57 62
58#define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver" 63#define NTB_NAME "ntb_hw_intel"
59#define NTB_VER "1.0" 64#define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
65#define NTB_VER "2.0"
60 66
61MODULE_DESCRIPTION(NTB_NAME); 67MODULE_DESCRIPTION(NTB_DESC);
62MODULE_VERSION(NTB_VER); 68MODULE_VERSION(NTB_VER);
63MODULE_LICENSE("Dual BSD/GPL"); 69MODULE_LICENSE("Dual BSD/GPL");
64MODULE_AUTHOR("Intel Corporation"); 70MODULE_AUTHOR("Intel Corporation");
65 71
66enum { 72#define bar0_off(base, bar) ((base) + ((bar) << 2))
67 NTB_CONN_TRANSPARENT = 0, 73#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
68 NTB_CONN_B2B, 74
69 NTB_CONN_RP, 75static int b2b_mw_idx = -1;
70}; 76module_param(b2b_mw_idx, int, 0644);
71 77MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
72enum { 78 "value of zero or positive starts from first mw idx, and a "
73 NTB_DEV_USD = 0, 79 "negative value starts from last mw idx. Both sides MUST "
74 NTB_DEV_DSD, 80 "set the same value here!");
75}; 81
76 82static unsigned int b2b_mw_share;
77enum { 83module_param(b2b_mw_share, uint, 0644);
78 SNB_HW = 0, 84MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
79 BWD_HW, 85 "ntb so that the peer ntb only occupies the first half of "
80}; 86 "the mw, so the second half can still be used as a mw. Both "
81 87 "sides MUST set the same value here!");
88
89static const struct intel_ntb_reg bwd_reg;
90static const struct intel_ntb_alt_reg bwd_pri_reg;
91static const struct intel_ntb_alt_reg bwd_sec_reg;
92static const struct intel_ntb_alt_reg bwd_b2b_reg;
93static const struct intel_ntb_xlat_reg bwd_pri_xlat;
94static const struct intel_ntb_xlat_reg bwd_sec_xlat;
95static const struct intel_ntb_reg snb_reg;
96static const struct intel_ntb_alt_reg snb_pri_reg;
97static const struct intel_ntb_alt_reg snb_sec_reg;
98static const struct intel_ntb_alt_reg snb_b2b_reg;
99static const struct intel_ntb_xlat_reg snb_pri_xlat;
100static const struct intel_ntb_xlat_reg snb_sec_xlat;
101static const struct intel_b2b_addr snb_b2b_usd_addr;
102static const struct intel_b2b_addr snb_b2b_dsd_addr;
103
104static const struct ntb_dev_ops intel_ntb_ops;
105
106static const struct file_operations intel_ntb_debugfs_info;
82static struct dentry *debugfs_dir; 107static struct dentry *debugfs_dir;
83 108
84#define BWD_LINK_RECOVERY_TIME 500 109#ifndef ioread64
85 110#ifdef readq
86/* Translate memory window 0,1,2 to BAR 2,4,5 */ 111#define ioread64 readq
87#define MW_TO_BAR(mw) (mw == 0 ? 2 : (mw == 1 ? 4 : 5)) 112#else
88 113#define ioread64 _ioread64
89static const struct pci_device_id ntb_pci_tbl[] = { 114static inline u64 _ioread64(void __iomem *mmio)
90 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
91 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
92 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
93 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
94 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
95 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
96 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
97 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
98 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
99 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
100 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
101 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
102 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
103 {0}
104};
105MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
106
107static int is_ntb_xeon(struct ntb_device *ndev)
108{ 115{
109 switch (ndev->pdev->device) { 116 u64 low, high;
110 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
111 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
112 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
113 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
114 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
115 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
116 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
117 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
118 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
119 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
120 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
121 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
122 return 1;
123 default:
124 return 0;
125 }
126 117
127 return 0; 118 low = ioread32(mmio);
119 high = ioread32(mmio + sizeof(u32));
120 return low | (high << 32);
121}
122#endif
123#endif
124
125#ifndef iowrite64
126#ifdef writeq
127#define iowrite64 writeq
128#else
129#define iowrite64 _iowrite64
130static inline void _iowrite64(u64 val, void __iomem *mmio)
131{
132 iowrite32(val, mmio);
133 iowrite32(val >> 32, mmio + sizeof(u32));
128} 134}
135#endif
136#endif
129 137
130static int is_ntb_atom(struct ntb_device *ndev) 138static inline int pdev_is_bwd(struct pci_dev *pdev)
131{ 139{
132 switch (ndev->pdev->device) { 140 switch (pdev->device) {
133 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD: 141 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
134 return 1; 142 return 1;
135 default:
136 return 0;
137 } 143 }
138
139 return 0; 144 return 0;
140} 145}
141 146
142static void ntb_set_errata_flags(struct ntb_device *ndev) 147static inline int pdev_is_snb(struct pci_dev *pdev)
143{ 148{
144 switch (ndev->pdev->device) { 149 switch (pdev->device) {
145 /*
146 * this workaround applies to all platform up to IvyBridge
147 * Haswell has splitbar support and use a different workaround
148 */
149 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: 150 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
150 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: 151 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
151 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 152 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
@@ -158,1737 +159,1957 @@ static void ntb_set_errata_flags(struct ntb_device *ndev)
158 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: 159 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
159 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 160 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
160 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 161 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
161 ndev->wa_flags |= WA_SNB_ERR; 162 return 1;
162 break;
163 } 163 }
164 return 0;
164} 165}
165 166
166/** 167static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
167 * ntb_register_event_callback() - register event callback
168 * @ndev: pointer to ntb_device instance
169 * @func: callback function to register
170 *
171 * This function registers a callback for any HW driver events such as link
172 * up/down, power management notices and etc.
173 *
174 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
175 */
176int ntb_register_event_callback(struct ntb_device *ndev,
177 void (*func)(void *handle,
178 enum ntb_hw_event event))
179{ 168{
180 if (ndev->event_cb) 169 ndev->unsafe_flags = 0;
181 return -EINVAL; 170 ndev->unsafe_flags_ignore = 0;
182 171
183 ndev->event_cb = func; 172 /* Only B2B has a workaround to avoid SDOORBELL */
184 173 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
185 return 0; 174 if (!ntb_topo_is_b2b(ndev->ntb.topo))
175 ndev->unsafe_flags |= NTB_UNSAFE_DB;
176
177 /* No low level workaround to avoid SB01BASE */
178 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
179 ndev->unsafe_flags |= NTB_UNSAFE_DB;
180 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
181 }
186} 182}
187 183
188/** 184static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
189 * ntb_unregister_event_callback() - unregisters the event callback 185 unsigned long flag)
190 * @ndev: pointer to ntb_device instance
191 *
192 * This function unregisters the existing callback from transport
193 */
194void ntb_unregister_event_callback(struct ntb_device *ndev)
195{ 186{
196 ndev->event_cb = NULL; 187 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
197} 188}
198 189
199static void ntb_irq_work(unsigned long data) 190static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
191 unsigned long flag)
200{ 192{
201 struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data; 193 flag &= ndev->unsafe_flags;
202 int rc; 194 ndev->unsafe_flags_ignore |= flag;
203 195
204 rc = db_cb->callback(db_cb->data, db_cb->db_num); 196 return !!flag;
205 if (rc)
206 tasklet_schedule(&db_cb->irq_work);
207 else {
208 struct ntb_device *ndev = db_cb->ndev;
209 unsigned long mask;
210
211 mask = readw(ndev->reg_ofs.ldb_mask);
212 clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
213 writew(mask, ndev->reg_ofs.ldb_mask);
214 }
215} 197}
216 198
217/** 199static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
218 * ntb_register_db_callback() - register a callback for doorbell interrupt
219 * @ndev: pointer to ntb_device instance
220 * @idx: doorbell index to register callback, zero based
221 * @data: pointer to be returned to caller with every callback
222 * @func: callback function to register
223 *
224 * This function registers a callback function for the doorbell interrupt
225 * on the primary side. The function will unmask the doorbell as well to
226 * allow interrupt.
227 *
228 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
229 */
230int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
231 void *data, int (*func)(void *data, int db_num))
232{ 200{
233 unsigned long mask; 201 if (idx < 0 || idx > ndev->mw_count)
234
235 if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
236 dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
237 return -EINVAL; 202 return -EINVAL;
238 } 203 return ndev->reg->mw_bar[idx];
204}
239 205
240 ndev->db_cb[idx].callback = func; 206static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
241 ndev->db_cb[idx].data = data; 207 phys_addr_t *db_addr, resource_size_t *db_size,
242 ndev->db_cb[idx].ndev = ndev; 208 phys_addr_t reg_addr, unsigned long reg)
209{
210 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
243 211
244 tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work, 212 if (db_addr) {
245 (unsigned long) &ndev->db_cb[idx]); 213 *db_addr = reg_addr + reg;
214 dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr);
215 }
246 216
247 /* unmask interrupt */ 217 if (db_size) {
248 mask = readw(ndev->reg_ofs.ldb_mask); 218 *db_size = ndev->reg->db_size;
249 clear_bit(idx * ndev->bits_per_vector, &mask); 219 dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size);
250 writew(mask, ndev->reg_ofs.ldb_mask); 220 }
251 221
252 return 0; 222 return 0;
253} 223}
254 224
255/** 225static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
256 * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt 226 void __iomem *mmio)
257 * @ndev: pointer to ntb_device instance
258 * @idx: doorbell index to register callback, zero based
259 *
260 * This function unregisters a callback function for the doorbell interrupt
261 * on the primary side. The function will also mask the said doorbell.
262 */
263void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
264{ 227{
265 unsigned long mask; 228 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
266 229
267 if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback) 230 return ndev->reg->db_ioread(mmio);
268 return; 231}
269 232
270 mask = readw(ndev->reg_ofs.ldb_mask); 233static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
271 set_bit(idx * ndev->bits_per_vector, &mask); 234 void __iomem *mmio)
272 writew(mask, ndev->reg_ofs.ldb_mask); 235{
236 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
273 237
274 tasklet_disable(&ndev->db_cb[idx].irq_work); 238 if (db_bits & ~ndev->db_valid_mask)
239 return -EINVAL;
275 240
276 ndev->db_cb[idx].callback = NULL; 241 ndev->reg->db_iowrite(db_bits, mmio);
277}
278 242
279/** 243 return 0;
280 * ntb_find_transport() - find the transport pointer
281 * @transport: pointer to pci device
282 *
283 * Given the pci device pointer, return the transport pointer passed in when
284 * the transport attached when it was inited.
285 *
286 * RETURNS: pointer to transport.
287 */
288void *ntb_find_transport(struct pci_dev *pdev)
289{
290 struct ntb_device *ndev = pci_get_drvdata(pdev);
291 return ndev->ntb_transport;
292} 244}
293 245
294/** 246static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
295 * ntb_register_transport() - Register NTB transport with NTB HW driver 247 void __iomem *mmio)
296 * @transport: transport identifier
297 *
298 * This function allows a transport to reserve the hardware driver for
299 * NTB usage.
300 *
301 * RETURNS: pointer to ntb_device, NULL on error.
302 */
303struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
304{ 248{
305 struct ntb_device *ndev = pci_get_drvdata(pdev); 249 unsigned long irqflags;
306 250
307 if (ndev->ntb_transport) 251 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
308 return NULL;
309 252
310 ndev->ntb_transport = transport; 253 if (db_bits & ~ndev->db_valid_mask)
311 return ndev; 254 return -EINVAL;
255
256 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
257 {
258 ndev->db_mask |= db_bits;
259 ndev->reg->db_iowrite(ndev->db_mask, mmio);
260 }
261 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
262
263 return 0;
312} 264}
313 265
314/** 266static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
315 * ntb_unregister_transport() - Unregister the transport with the NTB HW driver 267 void __iomem *mmio)
316 * @ndev - ntb_device of the transport to be freed
317 *
318 * This function unregisters the transport from the HW driver and performs any
319 * necessary cleanups.
320 */
321void ntb_unregister_transport(struct ntb_device *ndev)
322{ 268{
323 int i; 269 unsigned long irqflags;
324 270
325 if (!ndev->ntb_transport) 271 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
326 return; 272
273 if (db_bits & ~ndev->db_valid_mask)
274 return -EINVAL;
327 275
328 for (i = 0; i < ndev->max_cbs; i++) 276 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
329 ntb_unregister_db_callback(ndev, i); 277 {
278 ndev->db_mask &= ~db_bits;
279 ndev->reg->db_iowrite(ndev->db_mask, mmio);
280 }
281 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
330 282
331 ntb_unregister_event_callback(ndev); 283 return 0;
332 ndev->ntb_transport = NULL;
333} 284}
334 285
335/** 286static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
336 * ntb_write_local_spad() - write to the secondary scratchpad register
337 * @ndev: pointer to ntb_device instance
338 * @idx: index to the scratchpad register, 0 based
339 * @val: the data value to put into the register
340 *
341 * This function allows writing of a 32bit value to the indexed scratchpad
342 * register. This writes over the data mirrored to the local scratchpad register
343 * by the remote system.
344 *
345 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
346 */
347int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
348{ 287{
349 if (idx >= ndev->limits.max_spads) 288 u64 shift, mask;
350 return -EINVAL;
351 289
352 dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n", 290 shift = ndev->db_vec_shift;
353 val, idx); 291 mask = BIT_ULL(shift) - 1;
354 writel(val, ndev->reg_ofs.spad_read + idx * 4);
355 292
356 return 0; 293 return mask << (shift * db_vector);
357} 294}
358 295
359/** 296static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
360 * ntb_read_local_spad() - read from the primary scratchpad register 297 phys_addr_t *spad_addr, phys_addr_t reg_addr,
361 * @ndev: pointer to ntb_device instance 298 unsigned long reg)
362 * @idx: index to scratchpad register, 0 based
363 * @val: pointer to 32bit integer for storing the register value
364 *
365 * This function allows reading of the 32bit scratchpad register on
366 * the primary (internal) side. This allows the local system to read data
367 * written and mirrored to the scratchpad register by the remote system.
368 *
369 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
370 */
371int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
372{ 299{
373 if (idx >= ndev->limits.max_spads) 300 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
301
302 if (idx < 0 || idx >= ndev->spad_count)
374 return -EINVAL; 303 return -EINVAL;
375 304
376 *val = readl(ndev->reg_ofs.spad_write + idx * 4); 305 if (spad_addr) {
377 dev_dbg(&ndev->pdev->dev, 306 *spad_addr = reg_addr + reg + (idx << 2);
378 "Reading %x from local scratch pad index %d\n", *val, idx); 307 dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr);
308 }
379 309
380 return 0; 310 return 0;
381} 311}
382 312
383/** 313static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
384 * ntb_write_remote_spad() - write to the secondary scratchpad register 314 void __iomem *mmio)
385 * @ndev: pointer to ntb_device instance
386 * @idx: index to the scratchpad register, 0 based
387 * @val: the data value to put into the register
388 *
389 * This function allows writing of a 32bit value to the indexed scratchpad
390 * register. The register resides on the secondary (external) side. This allows
391 * the local system to write data to be mirrored to the remote systems
392 * scratchpad register.
393 *
394 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
395 */
396int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
397{ 315{
398 if (idx >= ndev->limits.max_spads) 316 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
399 return -EINVAL;
400 317
401 dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n", 318 if (idx < 0 || idx >= ndev->spad_count)
402 val, idx); 319 return 0;
403 writel(val, ndev->reg_ofs.spad_write + idx * 4);
404 320
405 return 0; 321 return ioread32(mmio + (idx << 2));
406} 322}
407 323
408/** 324static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
409 * ntb_read_remote_spad() - read from the primary scratchpad register 325 void __iomem *mmio)
410 * @ndev: pointer to ntb_device instance
411 * @idx: index to scratchpad register, 0 based
412 * @val: pointer to 32bit integer for storing the register value
413 *
414 * This function allows reading of the 32bit scratchpad register on
415 * the primary (internal) side. This alloows the local system to read the data
416 * it wrote to be mirrored on the remote system.
417 *
418 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
419 */
420int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
421{ 326{
422 if (idx >= ndev->limits.max_spads) 327 WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
328
329 if (idx < 0 || idx >= ndev->spad_count)
423 return -EINVAL; 330 return -EINVAL;
424 331
425 *val = readl(ndev->reg_ofs.spad_read + idx * 4); 332 iowrite32(val, mmio + (idx << 2));
426 dev_dbg(&ndev->pdev->dev,
427 "Reading %x from remote scratch pad index %d\n", *val, idx);
428 333
429 return 0; 334 return 0;
430} 335}
431 336
432/** 337static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
433 * ntb_get_mw_base() - get addr for the NTB memory window
434 * @ndev: pointer to ntb_device instance
435 * @mw: memory window number
436 *
437 * This function provides the base address of the memory window specified.
438 *
439 * RETURNS: address, or NULL on error.
440 */
441resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
442{ 338{
443 if (mw >= ntb_max_mw(ndev)) 339 u64 vec_mask;
444 return 0; 340
341 vec_mask = ndev_vec_mask(ndev, vec);
342
343 dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask);
344
345 ndev->last_ts = jiffies;
445 346
446 return pci_resource_start(ndev->pdev, MW_TO_BAR(mw)); 347 if (vec_mask & ndev->db_link_mask) {
348 if (ndev->reg->poll_link(ndev))
349 ntb_link_event(&ndev->ntb);
350 }
351
352 if (vec_mask & ndev->db_valid_mask)
353 ntb_db_event(&ndev->ntb, vec);
354
355 return IRQ_HANDLED;
447} 356}
448 357
449/** 358static irqreturn_t ndev_vec_isr(int irq, void *dev)
450 * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
451 * @ndev: pointer to ntb_device instance
452 * @mw: memory window number
453 *
454 * This function provides the base virtual address of the memory window
455 * specified.
456 *
457 * RETURNS: pointer to virtual address, or NULL on error.
458 */
459void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
460{ 359{
461 if (mw >= ntb_max_mw(ndev)) 360 struct intel_ntb_vec *nvec = dev;
462 return NULL;
463 361
464 return ndev->mw[mw].vbase; 362 return ndev_interrupt(nvec->ndev, nvec->num);
465} 363}
466 364
467/** 365static irqreturn_t ndev_irq_isr(int irq, void *dev)
468 * ntb_get_mw_size() - return size of NTB memory window
469 * @ndev: pointer to ntb_device instance
470 * @mw: memory window number
471 *
472 * This function provides the physical size of the memory window specified
473 *
474 * RETURNS: the size of the memory window or zero on error
475 */
476u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
477{ 366{
478 if (mw >= ntb_max_mw(ndev)) 367 struct intel_ntb_dev *ndev = dev;
479 return 0;
480 368
481 return ndev->mw[mw].bar_sz; 369 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
482} 370}
483 371
484/** 372static int ndev_init_isr(struct intel_ntb_dev *ndev,
485 * ntb_set_mw_addr - set the memory window address 373 int msix_min, int msix_max,
486 * @ndev: pointer to ntb_device instance 374 int msix_shift, int total_shift)
487 * @mw: memory window number
488 * @addr: base address for data
489 *
490 * This function sets the base physical address of the memory window. This
491 * memory address is where data from the remote system will be transfered into
492 * or out of depending on how the transport is configured.
493 */
494void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
495{ 375{
496 if (mw >= ntb_max_mw(ndev)) 376 struct pci_dev *pdev;
497 return; 377 int rc, i, msix_count;
498 378
499 dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr, 379 pdev = ndev_pdev(ndev);
500 MW_TO_BAR(mw));
501 380
502 ndev->mw[mw].phys_addr = addr; 381 /* Mask all doorbell interrupts */
382 ndev->db_mask = ndev->db_valid_mask;
383 ndev->reg->db_iowrite(ndev->db_mask,
384 ndev->self_mmio +
385 ndev->self_reg->db_mask);
503 386
504 switch (MW_TO_BAR(mw)) { 387 /* Try to set up msix irq */
505 case NTB_BAR_23: 388
506 writeq(addr, ndev->reg_ofs.bar2_xlat); 389 ndev->vec = kcalloc(msix_max, sizeof(*ndev->vec), GFP_KERNEL);
507 break; 390 if (!ndev->vec)
508 case NTB_BAR_4: 391 goto err_msix_vec_alloc;
509 if (ndev->split_bar) 392
510 writel(addr, ndev->reg_ofs.bar4_xlat); 393 ndev->msix = kcalloc(msix_max, sizeof(*ndev->msix), GFP_KERNEL);
511 else 394 if (!ndev->msix)
512 writeq(addr, ndev->reg_ofs.bar4_xlat); 395 goto err_msix_alloc;
513 break; 396
514 case NTB_BAR_5: 397 for (i = 0; i < msix_max; ++i)
515 writel(addr, ndev->reg_ofs.bar5_xlat); 398 ndev->msix[i].entry = i;
516 break; 399
400 msix_count = pci_enable_msix_range(pdev, ndev->msix,
401 msix_min, msix_max);
402 if (msix_count < 0)
403 goto err_msix_enable;
404
405 for (i = 0; i < msix_count; ++i) {
406 ndev->vec[i].ndev = ndev;
407 ndev->vec[i].num = i;
408 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
409 "ndev_vec_isr", &ndev->vec[i]);
410 if (rc)
411 goto err_msix_request;
517 } 412 }
518}
519 413
520/** 414 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
521 * ntb_ring_doorbell() - Set the doorbell on the secondary/external side 415 ndev->db_vec_count = msix_count;
522 * @ndev: pointer to ntb_device instance 416 ndev->db_vec_shift = msix_shift;
523 * @db: doorbell to ring 417 return 0;
524 *
525 * This function allows triggering of a doorbell on the secondary/external
526 * side that will initiate an interrupt on the remote host
527 *
528 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
529 */
530void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
531{
532 dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
533 418
534 if (ndev->hw_type == BWD_HW) 419err_msix_request:
535 writeq((u64) 1 << db, ndev->reg_ofs.rdb); 420 while (i-- > 0)
536 else 421 free_irq(ndev->msix[i].vector, ndev);
537 writew(((1 << ndev->bits_per_vector) - 1) << 422 pci_disable_msix(pdev);
538 (db * ndev->bits_per_vector), ndev->reg_ofs.rdb); 423err_msix_enable:
539} 424 kfree(ndev->msix);
425err_msix_alloc:
426 kfree(ndev->vec);
427err_msix_vec_alloc:
428 ndev->msix = NULL;
429 ndev->vec = NULL;
540 430
541static void bwd_recover_link(struct ntb_device *ndev) 431 /* Try to set up msi irq */
542{
543 u32 status;
544 432
545 /* Driver resets the NTB ModPhy lanes - magic! */ 433 rc = pci_enable_msi(pdev);
546 writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6); 434 if (rc)
547 writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4); 435 goto err_msi_enable;
548 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
549 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
550 436
551 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 437 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
552 msleep(100); 438 "ndev_irq_isr", ndev);
439 if (rc)
440 goto err_msi_request;
553 441
554 /* Clear AER Errors, write to clear */ 442 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
555 status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET); 443 ndev->db_vec_count = 1;
556 dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status); 444 ndev->db_vec_shift = total_shift;
557 status &= PCI_ERR_COR_REP_ROLL; 445 return 0;
558 writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
559 446
560 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 447err_msi_request:
561 status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET); 448 pci_disable_msi(pdev);
562 dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status); 449err_msi_enable:
563 status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
564 writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
565 450
566 /* Clear DeSkew Buffer error, write to clear */ 451 /* Try to set up intx irq */
567 status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
568 dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
569 status |= BWD_DESKEWSTS_DBERR;
570 writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
571 452
572 status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET); 453 pci_intx(pdev, 1);
573 dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
574 status &= BWD_IBIST_ERR_OFLOW;
575 writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
576 454
577 /* Releases the NTB state machine to allow the link to retrain */ 455 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
578 status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); 456 "ndev_irq_isr", ndev);
579 dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status); 457 if (rc)
580 status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT; 458 goto err_intx_request;
581 writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); 459
460 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
461 ndev->db_vec_count = 1;
462 ndev->db_vec_shift = total_shift;
463 return 0;
464
465err_intx_request:
466 return rc;
582} 467}
583 468
584static void ntb_link_event(struct ntb_device *ndev, int link_state) 469static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
585{ 470{
586 unsigned int event; 471 struct pci_dev *pdev;
472 int i;
587 473
588 if (ndev->link_status == link_state) 474 pdev = ndev_pdev(ndev);
589 return;
590 475
591 if (link_state == NTB_LINK_UP) { 476 /* Mask all doorbell interrupts */
592 u16 status; 477 ndev->db_mask = ndev->db_valid_mask;
593 478 ndev->reg->db_iowrite(ndev->db_mask,
594 dev_info(&ndev->pdev->dev, "Link Up\n"); 479 ndev->self_mmio +
595 ndev->link_status = NTB_LINK_UP; 480 ndev->self_reg->db_mask);
596 event = NTB_EVENT_HW_LINK_UP;
597
598 if (is_ntb_atom(ndev) ||
599 ndev->conn_type == NTB_CONN_TRANSPARENT)
600 status = readw(ndev->reg_ofs.lnk_stat);
601 else {
602 int rc = pci_read_config_word(ndev->pdev,
603 SNB_LINK_STATUS_OFFSET,
604 &status);
605 if (rc)
606 return;
607 }
608 481
609 ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4; 482 if (ndev->msix) {
610 ndev->link_speed = (status & NTB_LINK_SPEED_MASK); 483 i = ndev->db_vec_count;
611 dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n", 484 while (i--)
612 ndev->link_width, ndev->link_speed); 485 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
486 pci_disable_msix(pdev);
487 kfree(ndev->msix);
488 kfree(ndev->vec);
613 } else { 489 } else {
614 dev_info(&ndev->pdev->dev, "Link Down\n"); 490 free_irq(pdev->irq, ndev);
615 ndev->link_status = NTB_LINK_DOWN; 491 if (pci_dev_msi_enabled(pdev))
616 event = NTB_EVENT_HW_LINK_DOWN; 492 pci_disable_msi(pdev);
617 /* Don't modify link width/speed, we need it in link recovery */
618 } 493 }
619
620 /* notify the upper layer if we have an event change */
621 if (ndev->event_cb)
622 ndev->event_cb(ndev->ntb_transport, event);
623} 494}
624 495
625static int ntb_link_status(struct ntb_device *ndev) 496static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
497 size_t count, loff_t *offp)
626{ 498{
627 int link_state; 499 struct intel_ntb_dev *ndev;
500 void __iomem *mmio;
501 char *buf;
502 size_t buf_size;
503 ssize_t ret, off;
504 union { u64 v64; u32 v32; u16 v16; } u;
628 505
629 if (is_ntb_atom(ndev)) { 506 ndev = filp->private_data;
630 u32 ntb_cntl; 507 mmio = ndev->self_mmio;
631 508
632 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); 509 buf_size = min(count, 0x800ul);
633 if (ntb_cntl & BWD_CNTL_LINK_DOWN)
634 link_state = NTB_LINK_DOWN;
635 else
636 link_state = NTB_LINK_UP;
637 } else {
638 u16 status;
639 int rc;
640 510
641 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET, 511 buf = kmalloc(buf_size, GFP_KERNEL);
642 &status); 512 if (!buf)
643 if (rc) 513 return -ENOMEM;
644 return rc;
645 514
646 if (status & NTB_LINK_STATUS_ACTIVE) 515 off = 0;
647 link_state = NTB_LINK_UP; 516
648 else 517 off += scnprintf(buf + off, buf_size - off,
649 link_state = NTB_LINK_DOWN; 518 "NTB Device Information:\n");
519
520 off += scnprintf(buf + off, buf_size - off,
521 "Connection Topology -\t%s\n",
522 ntb_topo_string(ndev->ntb.topo));
523
524 off += scnprintf(buf + off, buf_size - off,
525 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
526 off += scnprintf(buf + off, buf_size - off,
527 "B2B MW Idx -\t\t%d\n", ndev->b2b_idx);
528 off += scnprintf(buf + off, buf_size - off,
529 "BAR4 Split -\t\t%s\n",
530 ndev->bar4_split ? "yes" : "no");
531
532 off += scnprintf(buf + off, buf_size - off,
533 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
534 off += scnprintf(buf + off, buf_size - off,
535 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
536
537 if (!ndev->reg->link_is_up(ndev)) {
538 off += scnprintf(buf + off, buf_size - off,
539 "Link Status -\t\tDown\n");
540 } else {
541 off += scnprintf(buf + off, buf_size - off,
542 "Link Status -\t\tUp\n");
543 off += scnprintf(buf + off, buf_size - off,
544 "Link Speed -\t\tPCI-E Gen %u\n",
545 NTB_LNK_STA_SPEED(ndev->lnk_sta));
546 off += scnprintf(buf + off, buf_size - off,
547 "Link Width -\t\tx%u\n",
548 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
650 } 549 }
651 550
652 ntb_link_event(ndev, link_state); 551 off += scnprintf(buf + off, buf_size - off,
552 "Memory Window Count -\t%u\n", ndev->mw_count);
553 off += scnprintf(buf + off, buf_size - off,
554 "Scratchpad Count -\t%u\n", ndev->spad_count);
555 off += scnprintf(buf + off, buf_size - off,
556 "Doorbell Count -\t%u\n", ndev->db_count);
557 off += scnprintf(buf + off, buf_size - off,
558 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
559 off += scnprintf(buf + off, buf_size - off,
560 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
561
562 off += scnprintf(buf + off, buf_size - off,
563 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
564 off += scnprintf(buf + off, buf_size - off,
565 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
566 off += scnprintf(buf + off, buf_size - off,
567 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
568
569 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
570 off += scnprintf(buf + off, buf_size - off,
571 "Doorbell Mask -\t\t%#llx\n", u.v64);
572
573 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
574 off += scnprintf(buf + off, buf_size - off,
575 "Doorbell Bell -\t\t%#llx\n", u.v64);
576
577 off += scnprintf(buf + off, buf_size - off,
578 "\nNTB Incoming XLAT:\n");
579
580 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
581 off += scnprintf(buf + off, buf_size - off,
582 "XLAT23 -\t\t%#018llx\n", u.v64);
583
584 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
585 off += scnprintf(buf + off, buf_size - off,
586 "XLAT45 -\t\t%#018llx\n", u.v64);
587
588 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
589 off += scnprintf(buf + off, buf_size - off,
590 "LMT23 -\t\t\t%#018llx\n", u.v64);
591
592 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
593 off += scnprintf(buf + off, buf_size - off,
594 "LMT45 -\t\t\t%#018llx\n", u.v64);
595
596 if (pdev_is_snb(ndev->ntb.pdev)) {
597 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
598 off += scnprintf(buf + off, buf_size - off,
599 "\nNTB Outgoing B2B XLAT:\n");
600
601 u.v64 = ioread64(mmio + SNB_PBAR23XLAT_OFFSET);
602 off += scnprintf(buf + off, buf_size - off,
603 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
604
605 u.v64 = ioread64(mmio + SNB_PBAR45XLAT_OFFSET);
606 off += scnprintf(buf + off, buf_size - off,
607 "B2B XLAT45 -\t\t%#018llx\n", u.v64);
608
609 u.v64 = ioread64(mmio + SNB_PBAR23LMT_OFFSET);
610 off += scnprintf(buf + off, buf_size - off,
611 "B2B LMT23 -\t\t%#018llx\n", u.v64);
612
613 u.v64 = ioread64(mmio + SNB_PBAR45LMT_OFFSET);
614 off += scnprintf(buf + off, buf_size - off,
615 "B2B LMT45 -\t\t%#018llx\n", u.v64);
616
617 off += scnprintf(buf + off, buf_size - off,
618 "\nNTB Secondary BAR:\n");
619
620 u.v64 = ioread64(mmio + SNB_SBAR0BASE_OFFSET);
621 off += scnprintf(buf + off, buf_size - off,
622 "SBAR01 -\t\t%#018llx\n", u.v64);
623
624 u.v64 = ioread64(mmio + SNB_SBAR23BASE_OFFSET);
625 off += scnprintf(buf + off, buf_size - off,
626 "SBAR23 -\t\t%#018llx\n", u.v64);
627
628 u.v64 = ioread64(mmio + SNB_SBAR45BASE_OFFSET);
629 off += scnprintf(buf + off, buf_size - off,
630 "SBAR45 -\t\t%#018llx\n", u.v64);
631 }
632
633 off += scnprintf(buf + off, buf_size - off,
634 "\nSNB NTB Statistics:\n");
653 635
654 return 0; 636 u.v16 = ioread16(mmio + SNB_USMEMMISS_OFFSET);
637 off += scnprintf(buf + off, buf_size - off,
638 "Upstream Memory Miss -\t%u\n", u.v16);
639
640 off += scnprintf(buf + off, buf_size - off,
641 "\nSNB NTB Hardware Errors:\n");
642
643 if (!pci_read_config_word(ndev->ntb.pdev,
644 SNB_DEVSTS_OFFSET, &u.v16))
645 off += scnprintf(buf + off, buf_size - off,
646 "DEVSTS -\t\t%#06x\n", u.v16);
647
648 if (!pci_read_config_word(ndev->ntb.pdev,
649 SNB_LINK_STATUS_OFFSET, &u.v16))
650 off += scnprintf(buf + off, buf_size - off,
651 "LNKSTS -\t\t%#06x\n", u.v16);
652
653 if (!pci_read_config_dword(ndev->ntb.pdev,
654 SNB_UNCERRSTS_OFFSET, &u.v32))
655 off += scnprintf(buf + off, buf_size - off,
656 "UNCERRSTS -\t\t%#06x\n", u.v32);
657
658 if (!pci_read_config_dword(ndev->ntb.pdev,
659 SNB_CORERRSTS_OFFSET, &u.v32))
660 off += scnprintf(buf + off, buf_size - off,
661 "CORERRSTS -\t\t%#06x\n", u.v32);
662 }
663
664 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
665 kfree(buf);
666 return ret;
655} 667}
656 668
657static void bwd_link_recovery(struct work_struct *work) 669static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
658{ 670{
659 struct ntb_device *ndev = container_of(work, struct ntb_device, 671 if (!debugfs_dir) {
660 lr_timer.work); 672 ndev->debugfs_dir = NULL;
661 u32 status32; 673 ndev->debugfs_info = NULL;
662 674 } else {
663 bwd_recover_link(ndev); 675 ndev->debugfs_dir =
664 /* There is a potential race between the 2 NTB devices recovering at the 676 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
665 * same time. If the times are the same, the link will not recover and 677 if (!ndev->debugfs_dir)
666 * the driver will be stuck in this loop forever. Add a random interval 678 ndev->debugfs_info = NULL;
667 * to the recovery time to prevent this race. 679 else
668 */ 680 ndev->debugfs_info =
669 msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME); 681 debugfs_create_file("info", S_IRUSR,
670 682 ndev->debugfs_dir, ndev,
671 status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET); 683 &intel_ntb_debugfs_info);
672 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
673 goto retry;
674
675 status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
676 if (status32 & BWD_IBIST_ERR_OFLOW)
677 goto retry;
678
679 status32 = readl(ndev->reg_ofs.lnk_cntl);
680 if (!(status32 & BWD_CNTL_LINK_DOWN)) {
681 unsigned char speed, width;
682 u16 status16;
683
684 status16 = readw(ndev->reg_ofs.lnk_stat);
685 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
686 speed = (status16 & NTB_LINK_SPEED_MASK);
687 if (ndev->link_width != width || ndev->link_speed != speed)
688 goto retry;
689 } 684 }
685}
690 686
691 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); 687static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
692 return; 688{
689 debugfs_remove_recursive(ndev->debugfs_dir);
690}
693 691
694retry: 692static int intel_ntb_mw_count(struct ntb_dev *ntb)
695 schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT); 693{
694 return ntb_ndev(ntb)->mw_count;
696} 695}
697 696
698/* BWD doesn't have link status interrupt, poll on that platform */ 697static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
699static void bwd_link_poll(struct work_struct *work) 698 phys_addr_t *base,
699 resource_size_t *size,
700 resource_size_t *align,
701 resource_size_t *align_size)
700{ 702{
701 struct ntb_device *ndev = container_of(work, struct ntb_device, 703 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
702 hb_timer.work); 704 int bar;
703 unsigned long ts = jiffies;
704 705
705 /* If we haven't gotten an interrupt in a while, check the BWD link 706 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
706 * status bit 707 idx += 1;
707 */ 708
708 if (ts > ndev->last_ts + NTB_HB_TIMEOUT) { 709 bar = ndev_mw_to_bar(ndev, idx);
709 int rc = ntb_link_status(ndev); 710 if (bar < 0)
710 if (rc) 711 return bar;
711 dev_err(&ndev->pdev->dev,
712 "Error determining link status\n");
713
714 /* Check to see if a link error is the cause of the link down */
715 if (ndev->link_status == NTB_LINK_DOWN) {
716 u32 status32 = readl(ndev->reg_base +
717 BWD_LTSSMSTATEJMP_OFFSET);
718 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
719 schedule_delayed_work(&ndev->lr_timer, 0);
720 return;
721 }
722 }
723 }
724 712
725 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT); 713 if (base)
714 *base = pci_resource_start(ndev->ntb.pdev, bar) +
715 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
716
717 if (size)
718 *size = pci_resource_len(ndev->ntb.pdev, bar) -
719 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
720
721 if (align)
722 *align = pci_resource_len(ndev->ntb.pdev, bar);
723
724 if (align_size)
725 *align_size = 1;
726
727 return 0;
726} 728}
727 729
728static int ntb_xeon_setup(struct ntb_device *ndev) 730static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
731 dma_addr_t addr, resource_size_t size)
729{ 732{
730 switch (ndev->conn_type) { 733 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
731 case NTB_CONN_B2B: 734 unsigned long base_reg, xlat_reg, limit_reg;
732 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET; 735 resource_size_t bar_size, mw_size;
733 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET; 736 void __iomem *mmio;
734 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET; 737 u64 base, limit, reg_val;
735 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET; 738 int bar;
736 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
737 if (ndev->split_bar)
738 ndev->reg_ofs.bar5_xlat =
739 ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
740 ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
741 739
742 /* There is a Xeon hardware errata related to writes to 740 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
743 * SDOORBELL or B2BDOORBELL in conjunction with inbound access 741 idx += 1;
744 * to NTB MMIO Space, which may hang the system. To workaround
745 * this use the second memory window to access the interrupt and
746 * scratch pad registers on the remote system.
747 */
748 if (ndev->wa_flags & WA_SNB_ERR) {
749 if (!ndev->mw[ndev->limits.max_mw - 1].bar_sz)
750 return -EINVAL;
751
752 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
753 ndev->reg_ofs.spad_write =
754 ndev->mw[ndev->limits.max_mw - 1].vbase +
755 SNB_SPAD_OFFSET;
756 ndev->reg_ofs.rdb =
757 ndev->mw[ndev->limits.max_mw - 1].vbase +
758 SNB_PDOORBELL_OFFSET;
759
760 /* Set the Limit register to 4k, the minimum size, to
761 * prevent an illegal access
762 */
763 writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
764 SNB_PBAR4LMT_OFFSET);
765 /* HW errata on the Limit registers. They can only be
766 * written when the base register is 4GB aligned and
767 * < 32bit. This should already be the case based on
768 * the driver defaults, but write the Limit registers
769 * first just in case.
770 */
771
772 ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
773 } else {
774 /* HW Errata on bit 14 of b2bdoorbell register. Writes
775 * will not be mirrored to the remote system. Shrink
776 * the number of bits by one, since bit 14 is the last
777 * bit.
778 */
779 ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1;
780 ndev->reg_ofs.spad_write = ndev->reg_base +
781 SNB_B2B_SPAD_OFFSET;
782 ndev->reg_ofs.rdb = ndev->reg_base +
783 SNB_B2B_DOORBELL_OFFSET;
784
785 /* Disable the Limit register, just incase it is set to
786 * something silly. A 64bit write should handle it
787 * regardless of whether it has a split BAR or not.
788 */
789 writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
790 /* HW errata on the Limit registers. They can only be
791 * written when the base register is 4GB aligned and
792 * < 32bit. This should already be the case based on
793 * the driver defaults, but write the Limit registers
794 * first just in case.
795 */
796 if (ndev->split_bar)
797 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
798 else
799 ndev->limits.max_mw = SNB_MAX_MW;
800 }
801 742
802 /* The Xeon errata workaround requires setting SBAR Base 743 bar = ndev_mw_to_bar(ndev, idx);
803 * addresses to known values, so that the PBAR XLAT can be 744 if (bar < 0)
804 * pointed at SBAR0 of the remote system. 745 return bar;
805 */
806 if (ndev->dev_type == NTB_DEV_USD) {
807 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
808 SNB_PBAR2XLAT_OFFSET);
809 if (ndev->wa_flags & WA_SNB_ERR)
810 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
811 SNB_PBAR4XLAT_OFFSET);
812 else {
813 if (ndev->split_bar) {
814 writel(SNB_MBAR4_DSD_ADDR,
815 ndev->reg_base +
816 SNB_PBAR4XLAT_OFFSET);
817 writel(SNB_MBAR5_DSD_ADDR,
818 ndev->reg_base +
819 SNB_PBAR5XLAT_OFFSET);
820 } else
821 writeq(SNB_MBAR4_DSD_ADDR,
822 ndev->reg_base +
823 SNB_PBAR4XLAT_OFFSET);
824
825 /* B2B_XLAT_OFFSET is a 64bit register, but can
826 * only take 32bit writes
827 */
828 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
829 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
830 writel(SNB_MBAR01_DSD_ADDR >> 32,
831 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
832 }
833
834 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
835 SNB_SBAR0BASE_OFFSET);
836 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
837 SNB_SBAR2BASE_OFFSET);
838 if (ndev->split_bar) {
839 writel(SNB_MBAR4_USD_ADDR, ndev->reg_base +
840 SNB_SBAR4BASE_OFFSET);
841 writel(SNB_MBAR5_USD_ADDR, ndev->reg_base +
842 SNB_SBAR5BASE_OFFSET);
843 } else
844 writeq(SNB_MBAR4_USD_ADDR, ndev->reg_base +
845 SNB_SBAR4BASE_OFFSET);
846 } else {
847 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
848 SNB_PBAR2XLAT_OFFSET);
849 if (ndev->wa_flags & WA_SNB_ERR)
850 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
851 SNB_PBAR4XLAT_OFFSET);
852 else {
853 if (ndev->split_bar) {
854 writel(SNB_MBAR4_USD_ADDR,
855 ndev->reg_base +
856 SNB_PBAR4XLAT_OFFSET);
857 writel(SNB_MBAR5_USD_ADDR,
858 ndev->reg_base +
859 SNB_PBAR5XLAT_OFFSET);
860 } else
861 writeq(SNB_MBAR4_USD_ADDR,
862 ndev->reg_base +
863 SNB_PBAR4XLAT_OFFSET);
864
865 /*
866 * B2B_XLAT_OFFSET is a 64bit register, but can
867 * only take 32bit writes
868 */
869 writel(SNB_MBAR01_USD_ADDR & 0xffffffff,
870 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
871 writel(SNB_MBAR01_USD_ADDR >> 32,
872 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
873 }
874 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
875 SNB_SBAR0BASE_OFFSET);
876 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
877 SNB_SBAR2BASE_OFFSET);
878 if (ndev->split_bar) {
879 writel(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
880 SNB_SBAR4BASE_OFFSET);
881 writel(SNB_MBAR5_DSD_ADDR, ndev->reg_base +
882 SNB_SBAR5BASE_OFFSET);
883 } else
884 writeq(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
885 SNB_SBAR4BASE_OFFSET);
886 746
887 } 747 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
888 break; 748
889 case NTB_CONN_RP: 749 if (idx == ndev->b2b_idx)
890 if (ndev->wa_flags & WA_SNB_ERR) { 750 mw_size = bar_size - ndev->b2b_off;
891 dev_err(&ndev->pdev->dev, 751 else
892 "NTB-RP disabled due to hardware errata.\n"); 752 mw_size = bar_size;
893 return -EINVAL; 753
754 /* hardware requires that addr is aligned to bar size */
755 if (addr & (bar_size - 1))
756 return -EINVAL;
757
758 /* make sure the range fits in the usable mw size */
759 if (size > mw_size)
760 return -EINVAL;
761
762 mmio = ndev->self_mmio;
763 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
764 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
765 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
766
767 if (bar < 4 || !ndev->bar4_split) {
768 base = ioread64(mmio + base_reg);
769
770 /* Set the limit if supported, if size is not mw_size */
771 if (limit_reg && size != mw_size)
772 limit = base + size;
773 else
774 limit = 0;
775
776 /* set and verify setting the translation address */
777 iowrite64(addr, mmio + xlat_reg);
778 reg_val = ioread64(mmio + xlat_reg);
779 if (reg_val != addr) {
780 iowrite64(0, mmio + xlat_reg);
781 return -EIO;
894 } 782 }
895 783
896 /* Scratch pads need to have exclusive access from the primary 784 /* set and verify setting the limit */
897 * or secondary side. Halve the num spads so that each side can 785 iowrite64(limit, mmio + limit_reg);
898 * have an equal amount. 786 reg_val = ioread64(mmio + limit_reg);
899 */ 787 if (reg_val != limit) {
900 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2; 788 iowrite64(base, mmio + limit_reg);
901 ndev->limits.max_db_bits = SNB_MAX_DB_BITS; 789 iowrite64(0, mmio + xlat_reg);
902 /* Note: The SDOORBELL is the cause of the errata. You REALLY 790 return -EIO;
903 * don't want to touch it.
904 */
905 ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
906 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
907 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
908 /* Offset the start of the spads to correspond to whether it is
909 * primary or secondary
910 */
911 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
912 ndev->limits.max_spads * 4;
913 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
914 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
915 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
916 if (ndev->split_bar) {
917 ndev->reg_ofs.bar5_xlat =
918 ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
919 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
920 } else
921 ndev->limits.max_mw = SNB_MAX_MW;
922 break;
923 case NTB_CONN_TRANSPARENT:
924 if (ndev->wa_flags & WA_SNB_ERR) {
925 dev_err(&ndev->pdev->dev,
926 "NTB-TRANSPARENT disabled due to hardware errata.\n");
927 return -EINVAL;
928 } 791 }
792 } else {
793 /* split bar addr range must all be 32 bit */
794 if (addr & (~0ull << 32))
795 return -EINVAL;
796 if ((addr + size) & (~0ull << 32))
797 return -EINVAL;
929 798
930 /* Scratch pads need to have exclusive access from the primary 799 base = ioread32(mmio + base_reg);
931 * or secondary side. Halve the num spads so that each side can
932 * have an equal amount.
933 */
934 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
935 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
936 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
937 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
938 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
939 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
940 /* Offset the start of the spads to correspond to whether it is
941 * primary or secondary
942 */
943 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
944 ndev->limits.max_spads * 4;
945 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
946 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
947
948 if (ndev->split_bar) {
949 ndev->reg_ofs.bar5_xlat =
950 ndev->reg_base + SNB_PBAR5XLAT_OFFSET;
951 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
952 } else
953 ndev->limits.max_mw = SNB_MAX_MW;
954 break;
955 default:
956 /*
957 * we should never hit this. the detect function should've
958 * take cared of everything.
959 */
960 return -EINVAL;
961 }
962 800
963 ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET; 801 /* Set the limit if supported, if size is not mw_size */
964 ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET; 802 if (limit_reg && size != mw_size)
965 ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET; 803 limit = base + size;
804 else
805 limit = 0;
806
807 /* set and verify setting the translation address */
808 iowrite32(addr, mmio + xlat_reg);
809 reg_val = ioread32(mmio + xlat_reg);
810 if (reg_val != addr) {
811 iowrite32(0, mmio + xlat_reg);
812 return -EIO;
813 }
966 814
967 ndev->limits.msix_cnt = SNB_MSIX_CNT; 815 /* set and verify setting the limit */
968 ndev->bits_per_vector = SNB_DB_BITS_PER_VEC; 816 iowrite32(limit, mmio + limit_reg);
817 reg_val = ioread32(mmio + limit_reg);
818 if (reg_val != limit) {
819 iowrite32(base, mmio + limit_reg);
820 iowrite32(0, mmio + xlat_reg);
821 return -EIO;
822 }
823 }
969 824
970 return 0; 825 return 0;
971} 826}
972 827
973static int ntb_bwd_setup(struct ntb_device *ndev) 828static int intel_ntb_link_is_up(struct ntb_dev *ntb,
829 enum ntb_speed *speed,
830 enum ntb_width *width)
974{ 831{
975 int rc; 832 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
976 u32 val;
977
978 ndev->hw_type = BWD_HW;
979 833
980 rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val); 834 if (ndev->reg->link_is_up(ndev)) {
981 if (rc) 835 if (speed)
982 return rc; 836 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
983 837 if (width)
984 switch ((val & BWD_PPD_CONN_TYPE) >> 8) { 838 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
985 case NTB_CONN_B2B: 839 return 1;
986 ndev->conn_type = NTB_CONN_B2B; 840 } else {
987 break; 841 /* TODO MAYBE: is it possible to observe the link speed and
988 case NTB_CONN_RP: 842 * width while link is training? */
989 default: 843 if (speed)
990 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n"); 844 *speed = NTB_SPEED_NONE;
991 return -EINVAL; 845 if (width)
846 *width = NTB_WIDTH_NONE;
847 return 0;
992 } 848 }
849}
993 850
994 if (val & BWD_PPD_DEV_TYPE) 851static int intel_ntb_link_enable(struct ntb_dev *ntb,
995 ndev->dev_type = NTB_DEV_DSD; 852 enum ntb_speed max_speed,
996 else 853 enum ntb_width max_width)
997 ndev->dev_type = NTB_DEV_USD; 854{
855 struct intel_ntb_dev *ndev;
856 u32 ntb_ctl;
998 857
999 /* Initiate PCI-E link training */ 858 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1000 rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
1001 val | BWD_PPD_INIT_LINK);
1002 if (rc)
1003 return rc;
1004 859
1005 ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET; 860 if (ndev->ntb.topo == NTB_TOPO_SEC)
1006 ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET; 861 return -EINVAL;
1007 ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET; 862
1008 ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET; 863 dev_dbg(ndev_dev(ndev),
1009 ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET; 864 "Enabling link with max_speed %d max_width %d\n",
1010 ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET; 865 max_speed, max_width);
1011 ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET; 866 if (max_speed != NTB_SPEED_AUTO)
1012 ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET; 867 dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
1013 ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET; 868 if (max_width != NTB_WIDTH_AUTO)
1014 ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET; 869 dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
1015 ndev->limits.max_mw = BWD_MAX_MW; 870
1016 ndev->limits.max_spads = BWD_MAX_SPADS; 871 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1017 ndev->limits.max_db_bits = BWD_MAX_DB_BITS; 872 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
1018 ndev->limits.msix_cnt = BWD_MSIX_CNT; 873 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
1019 ndev->bits_per_vector = BWD_DB_BITS_PER_VEC; 874 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
1020 875 if (ndev->bar4_split)
1021 /* Since bwd doesn't have a link interrupt, setup a poll timer */ 876 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
1022 INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll); 877 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1023 INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
1024 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
1025 878
1026 return 0; 879 return 0;
1027} 880}
1028 881
1029static int ntb_device_setup(struct ntb_device *ndev) 882static int intel_ntb_link_disable(struct ntb_dev *ntb)
1030{ 883{
1031 int rc; 884 struct intel_ntb_dev *ndev;
885 u32 ntb_cntl;
1032 886
1033 if (is_ntb_xeon(ndev)) 887 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1034 rc = ntb_xeon_setup(ndev);
1035 else if (is_ntb_atom(ndev))
1036 rc = ntb_bwd_setup(ndev);
1037 else
1038 rc = -ENODEV;
1039 888
1040 if (rc) 889 if (ndev->ntb.topo == NTB_TOPO_SEC)
1041 return rc; 890 return -EINVAL;
1042 891
1043 if (ndev->conn_type == NTB_CONN_B2B) 892 dev_dbg(ndev_dev(ndev), "Disabling link\n");
1044 /* Enable Bus Master and Memory Space on the secondary side */ 893
1045 writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, 894 /* Bring NTB link down */
1046 ndev->reg_ofs.spci_cmd); 895 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
896 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
897 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
898 if (ndev->bar4_split)
899 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
900 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
901 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1047 902
1048 return 0; 903 return 0;
1049} 904}
1050 905
1051static void ntb_device_free(struct ntb_device *ndev) 906static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1052{ 907{
1053 if (is_ntb_atom(ndev)) { 908 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1054 cancel_delayed_work_sync(&ndev->hb_timer);
1055 cancel_delayed_work_sync(&ndev->lr_timer);
1056 }
1057} 909}
1058 910
1059static irqreturn_t bwd_callback_msix_irq(int irq, void *data) 911static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1060{ 912{
1061 struct ntb_db_cb *db_cb = data; 913 return ntb_ndev(ntb)->db_valid_mask;
1062 struct ntb_device *ndev = db_cb->ndev; 914}
1063 unsigned long mask;
1064 915
1065 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, 916static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1066 db_cb->db_num); 917{
918 struct intel_ntb_dev *ndev;
1067 919
1068 mask = readw(ndev->reg_ofs.ldb_mask); 920 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1069 set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
1070 writew(mask, ndev->reg_ofs.ldb_mask);
1071 921
1072 tasklet_schedule(&db_cb->irq_work); 922 return ndev->db_vec_count;
923}
1073 924
1074 /* No need to check for the specific HB irq, any interrupt means 925static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1075 * we're connected. 926{
1076 */ 927 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1077 ndev->last_ts = jiffies;
1078 928
1079 writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb); 929 if (db_vector < 0 || db_vector > ndev->db_vec_count)
930 return 0;
1080 931
1081 return IRQ_HANDLED; 932 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1082} 933}
1083 934
1084static irqreturn_t xeon_callback_msix_irq(int irq, void *data) 935static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1085{ 936{
1086 struct ntb_db_cb *db_cb = data; 937 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1087 struct ntb_device *ndev = db_cb->ndev;
1088 unsigned long mask;
1089 938
1090 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq, 939 return ndev_db_read(ndev,
1091 db_cb->db_num); 940 ndev->self_mmio +
941 ndev->self_reg->db_bell);
942}
1092 943
1093 mask = readw(ndev->reg_ofs.ldb_mask); 944static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1094 set_bit(db_cb->db_num * ndev->bits_per_vector, &mask); 945{
1095 writew(mask, ndev->reg_ofs.ldb_mask); 946 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1096 947
1097 tasklet_schedule(&db_cb->irq_work); 948 return ndev_db_write(ndev, db_bits,
949 ndev->self_mmio +
950 ndev->self_reg->db_bell);
951}
1098 952
1099 /* On Sandybridge, there are 16 bits in the interrupt register 953static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1100 * but only 4 vectors. So, 5 bits are assigned to the first 3 954{
1101 * vectors, with the 4th having a single bit for link 955 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1102 * interrupts.
1103 */
1104 writew(((1 << ndev->bits_per_vector) - 1) <<
1105 (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
1106 956
1107 return IRQ_HANDLED; 957 return ndev_db_set_mask(ndev, db_bits,
958 ndev->self_mmio +
959 ndev->self_reg->db_mask);
1108} 960}
1109 961
1110/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */ 962static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1111static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
1112{ 963{
1113 struct ntb_device *ndev = dev; 964 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1114 int rc;
1115
1116 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
1117 965
1118 rc = ntb_link_status(ndev); 966 return ndev_db_clear_mask(ndev, db_bits,
1119 if (rc) 967 ndev->self_mmio +
1120 dev_err(&ndev->pdev->dev, "Error determining link status\n"); 968 ndev->self_reg->db_mask);
969}
1121 970
1122 /* bit 15 is always the link bit */ 971static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
1123 writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb); 972 phys_addr_t *db_addr,
973 resource_size_t *db_size)
974{
975 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1124 976
1125 return IRQ_HANDLED; 977 return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
978 ndev->peer_reg->db_bell);
1126} 979}
1127 980
1128static irqreturn_t ntb_interrupt(int irq, void *dev) 981static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1129{ 982{
1130 struct ntb_device *ndev = dev; 983 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1131 unsigned int i = 0;
1132 984
1133 if (is_ntb_atom(ndev)) { 985 return ndev_db_write(ndev, db_bits,
1134 u64 ldb = readq(ndev->reg_ofs.ldb); 986 ndev->peer_mmio +
987 ndev->peer_reg->db_bell);
988}
1135 989
1136 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb); 990static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
991{
992 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
993}
1137 994
1138 while (ldb) { 995static int intel_ntb_spad_count(struct ntb_dev *ntb)
1139 i = __ffs(ldb); 996{
1140 ldb &= ldb - 1; 997 struct intel_ntb_dev *ndev;
1141 bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
1142 }
1143 } else {
1144 u16 ldb = readw(ndev->reg_ofs.ldb);
1145 998
1146 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb); 999 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1147 1000
1148 if (ldb & SNB_DB_HW_LINK) { 1001 return ndev->spad_count;
1149 xeon_event_msix_irq(irq, dev); 1002}
1150 ldb &= ~SNB_DB_HW_LINK;
1151 }
1152 1003
1153 while (ldb) { 1004static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1154 i = __ffs(ldb); 1005{
1155 ldb &= ldb - 1; 1006 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1156 xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
1157 }
1158 }
1159 1007
1160 return IRQ_HANDLED; 1008 return ndev_spad_read(ndev, idx,
1009 ndev->self_mmio +
1010 ndev->self_reg->spad);
1161} 1011}
1162 1012
1163static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries) 1013static int intel_ntb_spad_write(struct ntb_dev *ntb,
1014 int idx, u32 val)
1164{ 1015{
1165 struct pci_dev *pdev = ndev->pdev; 1016 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1166 struct msix_entry *msix;
1167 int rc, i;
1168 1017
1169 if (msix_entries < ndev->limits.msix_cnt) 1018 return ndev_spad_write(ndev, idx, val,
1170 return -ENOSPC; 1019 ndev->self_mmio +
1020 ndev->self_reg->spad);
1021}
1171 1022
1172 rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries); 1023static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
1173 if (rc < 0) 1024 phys_addr_t *spad_addr)
1174 return rc; 1025{
1026 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1175 1027
1176 for (i = 0; i < msix_entries; i++) { 1028 return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
1177 msix = &ndev->msix_entries[i]; 1029 ndev->peer_reg->spad);
1178 WARN_ON(!msix->vector); 1030}
1179 1031
1180 if (i == msix_entries - 1) { 1032static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
1181 rc = request_irq(msix->vector, 1033{
1182 xeon_event_msix_irq, 0, 1034 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1183 "ntb-event-msix", ndev);
1184 if (rc)
1185 goto err;
1186 } else {
1187 rc = request_irq(msix->vector,
1188 xeon_callback_msix_irq, 0,
1189 "ntb-callback-msix",
1190 &ndev->db_cb[i]);
1191 if (rc)
1192 goto err;
1193 }
1194 }
1195 1035
1196 ndev->num_msix = msix_entries; 1036 return ndev_spad_read(ndev, idx,
1197 ndev->max_cbs = msix_entries - 1; 1037 ndev->peer_mmio +
1038 ndev->peer_reg->spad);
1039}
1198 1040
1199 return 0; 1041static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
1042 int idx, u32 val)
1043{
1044 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1200 1045
1201err: 1046 return ndev_spad_write(ndev, idx, val,
1202 while (--i >= 0) { 1047 ndev->peer_mmio +
1203 /* Code never reaches here for entry nr 'ndev->num_msix - 1' */ 1048 ndev->peer_reg->spad);
1204 msix = &ndev->msix_entries[i]; 1049}
1205 free_irq(msix->vector, &ndev->db_cb[i]);
1206 }
1207 1050
1208 pci_disable_msix(pdev); 1051/* BWD */
1209 ndev->num_msix = 0;
1210 1052
1211 return rc; 1053static u64 bwd_db_ioread(void __iomem *mmio)
1054{
1055 return ioread64(mmio);
1212} 1056}
1213 1057
1214static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries) 1058static void bwd_db_iowrite(u64 bits, void __iomem *mmio)
1215{ 1059{
1216 struct pci_dev *pdev = ndev->pdev; 1060 iowrite64(bits, mmio);
1217 struct msix_entry *msix; 1061}
1218 int rc, i;
1219 1062
1220 msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries, 1063static int bwd_poll_link(struct intel_ntb_dev *ndev)
1221 1, msix_entries); 1064{
1222 if (msix_entries < 0) 1065 u32 ntb_ctl;
1223 return msix_entries;
1224 1066
1225 for (i = 0; i < msix_entries; i++) { 1067 ntb_ctl = ioread32(ndev->self_mmio + BWD_NTBCNTL_OFFSET);
1226 msix = &ndev->msix_entries[i];
1227 WARN_ON(!msix->vector);
1228 1068
1229 rc = request_irq(msix->vector, bwd_callback_msix_irq, 0, 1069 if (ntb_ctl == ndev->ntb_ctl)
1230 "ntb-callback-msix", &ndev->db_cb[i]); 1070 return 0;
1231 if (rc)
1232 goto err;
1233 }
1234 1071
1235 ndev->num_msix = msix_entries; 1072 ndev->ntb_ctl = ntb_ctl;
1236 ndev->max_cbs = msix_entries;
1237 1073
1238 return 0; 1074 ndev->lnk_sta = ioread32(ndev->self_mmio + BWD_LINK_STATUS_OFFSET);
1239 1075
1240err: 1076 return 1;
1241 while (--i >= 0) 1077}
1242 free_irq(msix->vector, &ndev->db_cb[i]);
1243 1078
1244 pci_disable_msix(pdev); 1079static int bwd_link_is_up(struct intel_ntb_dev *ndev)
1245 ndev->num_msix = 0; 1080{
1081 return BWD_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1082}
1246 1083
1247 return rc; 1084static int bwd_link_is_err(struct intel_ntb_dev *ndev)
1085{
1086 if (ioread32(ndev->self_mmio + BWD_LTSSMSTATEJMP_OFFSET)
1087 & BWD_LTSSMSTATEJMP_FORCEDETECT)
1088 return 1;
1089
1090 if (ioread32(ndev->self_mmio + BWD_IBSTERRRCRVSTS0_OFFSET)
1091 & BWD_IBIST_ERR_OFLOW)
1092 return 1;
1093
1094 return 0;
1248} 1095}
1249 1096
1250static int ntb_setup_msix(struct ntb_device *ndev) 1097static inline enum ntb_topo bwd_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1251{ 1098{
1252 struct pci_dev *pdev = ndev->pdev; 1099 switch (ppd & BWD_PPD_TOPO_MASK) {
1253 int msix_entries; 1100 case BWD_PPD_TOPO_B2B_USD:
1254 int rc, i; 1101 dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
1102 return NTB_TOPO_B2B_USD;
1103
1104 case BWD_PPD_TOPO_B2B_DSD:
1105 dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
1106 return NTB_TOPO_B2B_DSD;
1107
1108 case BWD_PPD_TOPO_PRI_USD:
1109 case BWD_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1110 case BWD_PPD_TOPO_SEC_USD:
1111 case BWD_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1112 dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
1113 return NTB_TOPO_NONE;
1114 }
1255 1115
1256 msix_entries = pci_msix_vec_count(pdev); 1116 dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd);
1257 if (msix_entries < 0) { 1117 return NTB_TOPO_NONE;
1258 rc = msix_entries; 1118}
1259 goto err; 1119
1260 } else if (msix_entries > ndev->limits.msix_cnt) { 1120static void bwd_link_hb(struct work_struct *work)
1261 rc = -EINVAL; 1121{
1262 goto err; 1122 struct intel_ntb_dev *ndev = hb_ndev(work);
1123 unsigned long poll_ts;
1124 void __iomem *mmio;
1125 u32 status32;
1126
1127 poll_ts = ndev->last_ts + BWD_LINK_HB_TIMEOUT;
1128
1129 /* Delay polling the link status if an interrupt was received,
1130 * unless the cached link status says the link is down.
1131 */
1132 if (time_after(poll_ts, jiffies) && bwd_link_is_up(ndev)) {
1133 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1134 return;
1263 } 1135 }
1264 1136
1265 ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries, 1137 if (bwd_poll_link(ndev))
1266 GFP_KERNEL); 1138 ntb_link_event(&ndev->ntb);
1267 if (!ndev->msix_entries) { 1139
1268 rc = -ENOMEM; 1140 if (bwd_link_is_up(ndev) || !bwd_link_is_err(ndev)) {
1269 goto err; 1141 schedule_delayed_work(&ndev->hb_timer, BWD_LINK_HB_TIMEOUT);
1142 return;
1270 } 1143 }
1271 1144
1272 for (i = 0; i < msix_entries; i++) 1145 /* Link is down with error: recover the link! */
1273 ndev->msix_entries[i].entry = i;
1274 1146
1275 if (is_ntb_atom(ndev)) 1147 mmio = ndev->self_mmio;
1276 rc = ntb_setup_bwd_msix(ndev, msix_entries);
1277 else
1278 rc = ntb_setup_snb_msix(ndev, msix_entries);
1279 if (rc)
1280 goto err1;
1281 1148
1282 return 0; 1149 /* Driver resets the NTB ModPhy lanes - magic! */
1150 iowrite8(0xe0, mmio + BWD_MODPHY_PCSREG6);
1151 iowrite8(0x40, mmio + BWD_MODPHY_PCSREG4);
1152 iowrite8(0x60, mmio + BWD_MODPHY_PCSREG4);
1153 iowrite8(0x60, mmio + BWD_MODPHY_PCSREG6);
1283 1154
1284err1: 1155 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1285 kfree(ndev->msix_entries); 1156 msleep(100);
1286err: 1157
1287 dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n"); 1158 /* Clear AER Errors, write to clear */
1288 return rc; 1159 status32 = ioread32(mmio + BWD_ERRCORSTS_OFFSET);
1160 dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32);
1161 status32 &= PCI_ERR_COR_REP_ROLL;
1162 iowrite32(status32, mmio + BWD_ERRCORSTS_OFFSET);
1163
1164 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1165 status32 = ioread32(mmio + BWD_LTSSMERRSTS0_OFFSET);
1166 dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32);
1167 status32 |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
1168 iowrite32(status32, mmio + BWD_LTSSMERRSTS0_OFFSET);
1169
1170 /* Clear DeSkew Buffer error, write to clear */
1171 status32 = ioread32(mmio + BWD_DESKEWSTS_OFFSET);
1172 dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32);
1173 status32 |= BWD_DESKEWSTS_DBERR;
1174 iowrite32(status32, mmio + BWD_DESKEWSTS_OFFSET);
1175
1176 status32 = ioread32(mmio + BWD_IBSTERRRCRVSTS0_OFFSET);
1177 dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32);
1178 status32 &= BWD_IBIST_ERR_OFLOW;
1179 iowrite32(status32, mmio + BWD_IBSTERRRCRVSTS0_OFFSET);
1180
1181 /* Releases the NTB state machine to allow the link to retrain */
1182 status32 = ioread32(mmio + BWD_LTSSMSTATEJMP_OFFSET);
1183 dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32);
1184 status32 &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
1185 iowrite32(status32, mmio + BWD_LTSSMSTATEJMP_OFFSET);
1186
1187 /* There is a potential race between the 2 NTB devices recovering at the
1188 * same time. If the times are the same, the link will not recover and
1189 * the driver will be stuck in this loop forever. Add a random interval
1190 * to the recovery time to prevent this race.
1191 */
1192 schedule_delayed_work(&ndev->hb_timer, BWD_LINK_RECOVERY_TIME
1193 + prandom_u32() % BWD_LINK_RECOVERY_TIME);
1289} 1194}
1290 1195
1291static int ntb_setup_msi(struct ntb_device *ndev) 1196static int bwd_init_isr(struct intel_ntb_dev *ndev)
1292{ 1197{
1293 struct pci_dev *pdev = ndev->pdev;
1294 int rc; 1198 int rc;
1295 1199
1296 rc = pci_enable_msi(pdev); 1200 rc = ndev_init_isr(ndev, 1, BWD_DB_MSIX_VECTOR_COUNT,
1201 BWD_DB_MSIX_VECTOR_SHIFT, BWD_DB_TOTAL_SHIFT);
1297 if (rc) 1202 if (rc)
1298 return rc; 1203 return rc;
1299 1204
1300 rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev); 1205 /* BWD doesn't have link status interrupt, poll on that platform */
1301 if (rc) { 1206 ndev->last_ts = jiffies;
1302 pci_disable_msi(pdev); 1207 INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_hb);
1303 dev_err(&pdev->dev, "Error allocating MSI interrupt\n"); 1208 schedule_delayed_work(&ndev->hb_timer, BWD_LINK_HB_TIMEOUT);
1304 return rc;
1305 }
1306 1209
1307 return 0; 1210 return 0;
1308} 1211}
1309 1212
1310static int ntb_setup_intx(struct ntb_device *ndev) 1213static void bwd_deinit_isr(struct intel_ntb_dev *ndev)
1311{ 1214{
1312 struct pci_dev *pdev = ndev->pdev; 1215 cancel_delayed_work_sync(&ndev->hb_timer);
1313 int rc; 1216 ndev_deinit_isr(ndev);
1217}
1314 1218
1315 /* Verify intx is enabled */ 1219static int bwd_init_ntb(struct intel_ntb_dev *ndev)
1316 pci_intx(pdev, 1); 1220{
1221 ndev->mw_count = BWD_MW_COUNT;
1222 ndev->spad_count = BWD_SPAD_COUNT;
1223 ndev->db_count = BWD_DB_COUNT;
1317 1224
1318 rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx", 1225 switch (ndev->ntb.topo) {
1319 ndev); 1226 case NTB_TOPO_B2B_USD:
1320 if (rc) 1227 case NTB_TOPO_B2B_DSD:
1321 return rc; 1228 ndev->self_reg = &bwd_pri_reg;
1229 ndev->peer_reg = &bwd_b2b_reg;
1230 ndev->xlat_reg = &bwd_sec_xlat;
1231
1232 /* Enable Bus Master and Memory Space on the secondary side */
1233 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1234 ndev->self_mmio + BWD_SPCICMD_OFFSET);
1235
1236 break;
1237
1238 default:
1239 return -EINVAL;
1240 }
1241
1242 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1322 1243
1323 return 0; 1244 return 0;
1324} 1245}
1325 1246
1326static int ntb_setup_interrupts(struct ntb_device *ndev) 1247static int bwd_init_dev(struct intel_ntb_dev *ndev)
1327{ 1248{
1249 u32 ppd;
1328 int rc; 1250 int rc;
1329 1251
1330 /* On BWD, disable all interrupts. On SNB, disable all but Link 1252 rc = pci_read_config_dword(ndev->ntb.pdev, BWD_PPD_OFFSET, &ppd);
1331 * Interrupt. The rest will be unmasked as callbacks are registered. 1253 if (rc)
1332 */ 1254 return -EIO;
1333 if (is_ntb_atom(ndev))
1334 writeq(~0, ndev->reg_ofs.ldb_mask);
1335 else {
1336 u16 var = 1 << SNB_LINK_DB;
1337 writew(~var, ndev->reg_ofs.ldb_mask);
1338 }
1339
1340 rc = ntb_setup_msix(ndev);
1341 if (!rc)
1342 goto done;
1343 1255
1344 ndev->bits_per_vector = 1; 1256 ndev->ntb.topo = bwd_ppd_topo(ndev, ppd);
1345 ndev->max_cbs = ndev->limits.max_db_bits; 1257 if (ndev->ntb.topo == NTB_TOPO_NONE)
1258 return -EINVAL;
1346 1259
1347 rc = ntb_setup_msi(ndev); 1260 rc = bwd_init_ntb(ndev);
1348 if (!rc) 1261 if (rc)
1349 goto done; 1262 return rc;
1350 1263
1351 rc = ntb_setup_intx(ndev); 1264 rc = bwd_init_isr(ndev);
1352 if (rc) { 1265 if (rc)
1353 dev_err(&ndev->pdev->dev, "no usable interrupts\n");
1354 return rc; 1266 return rc;
1267
1268 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1269 /* Initiate PCI-E link training */
1270 rc = pci_write_config_dword(ndev->ntb.pdev, BWD_PPD_OFFSET,
1271 ppd | BWD_PPD_INIT_LINK);
1272 if (rc)
1273 return rc;
1355 } 1274 }
1356 1275
1357done:
1358 return 0; 1276 return 0;
1359} 1277}
1360 1278
1361static void ntb_free_interrupts(struct ntb_device *ndev) 1279static void bwd_deinit_dev(struct intel_ntb_dev *ndev)
1362{ 1280{
1363 struct pci_dev *pdev = ndev->pdev; 1281 bwd_deinit_isr(ndev);
1282}
1364 1283
1365 /* mask interrupts */ 1284/* SNB */
1366 if (is_ntb_atom(ndev))
1367 writeq(~0, ndev->reg_ofs.ldb_mask);
1368 else
1369 writew(~0, ndev->reg_ofs.ldb_mask);
1370 1285
1371 if (ndev->num_msix) { 1286static u64 snb_db_ioread(void __iomem *mmio)
1372 struct msix_entry *msix; 1287{
1373 u32 i; 1288 return (u64)ioread16(mmio);
1289}
1374 1290
1375 for (i = 0; i < ndev->num_msix; i++) { 1291static void snb_db_iowrite(u64 bits, void __iomem *mmio)
1376 msix = &ndev->msix_entries[i]; 1292{
1377 if (is_ntb_xeon(ndev) && i == ndev->num_msix - 1) 1293 iowrite16((u16)bits, mmio);
1378 free_irq(msix->vector, ndev); 1294}
1379 else
1380 free_irq(msix->vector, &ndev->db_cb[i]);
1381 }
1382 pci_disable_msix(pdev);
1383 kfree(ndev->msix_entries);
1384 } else {
1385 free_irq(pdev->irq, ndev);
1386 1295
1387 if (pci_dev_msi_enabled(pdev)) 1296static int snb_poll_link(struct intel_ntb_dev *ndev)
1388 pci_disable_msi(pdev); 1297{
1389 } 1298 u16 reg_val;
1299 int rc;
1300
1301 ndev->reg->db_iowrite(ndev->db_link_mask,
1302 ndev->self_mmio +
1303 ndev->self_reg->db_bell);
1304
1305 rc = pci_read_config_word(ndev->ntb.pdev,
1306 SNB_LINK_STATUS_OFFSET, &reg_val);
1307 if (rc)
1308 return 0;
1309
1310 if (reg_val == ndev->lnk_sta)
1311 return 0;
1312
1313 ndev->lnk_sta = reg_val;
1314
1315 return 1;
1390} 1316}
1391 1317
1392static int ntb_create_callbacks(struct ntb_device *ndev) 1318static int snb_link_is_up(struct intel_ntb_dev *ndev)
1393{ 1319{
1394 int i; 1320 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1321}
1395 1322
1396 /* Chicken-egg issue. We won't know how many callbacks are necessary 1323static inline enum ntb_topo snb_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1397 * until we see how many MSI-X vectors we get, but these pointers need 1324{
1398 * to be passed into the MSI-X register function. So, we allocate the 1325 switch (ppd & SNB_PPD_TOPO_MASK) {
1399 * max, knowing that they might not all be used, to work around this. 1326 case SNB_PPD_TOPO_B2B_USD:
1400 */ 1327 return NTB_TOPO_B2B_USD;
1401 ndev->db_cb = kcalloc(ndev->limits.max_db_bits, 1328
1402 sizeof(struct ntb_db_cb), 1329 case SNB_PPD_TOPO_B2B_DSD:
1403 GFP_KERNEL); 1330 return NTB_TOPO_B2B_DSD;
1404 if (!ndev->db_cb) 1331
1405 return -ENOMEM; 1332 case SNB_PPD_TOPO_PRI_USD:
1333 case SNB_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1334 return NTB_TOPO_PRI;
1406 1335
1407 for (i = 0; i < ndev->limits.max_db_bits; i++) { 1336 case SNB_PPD_TOPO_SEC_USD:
1408 ndev->db_cb[i].db_num = i; 1337 case SNB_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1409 ndev->db_cb[i].ndev = ndev; 1338 return NTB_TOPO_SEC;
1410 } 1339 }
1411 1340
1412 return 0; 1341 return NTB_TOPO_NONE;
1413} 1342}
1414 1343
1415static void ntb_free_callbacks(struct ntb_device *ndev) 1344static inline int snb_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1416{ 1345{
1417 int i; 1346 if (ppd & SNB_PPD_SPLIT_BAR_MASK) {
1347 dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd);
1348 return 1;
1349 }
1350 return 0;
1351}
1418 1352
1419 for (i = 0; i < ndev->limits.max_db_bits; i++) 1353static int snb_init_isr(struct intel_ntb_dev *ndev)
1420 ntb_unregister_db_callback(ndev, i); 1354{
1355 return ndev_init_isr(ndev, SNB_DB_MSIX_VECTOR_COUNT,
1356 SNB_DB_MSIX_VECTOR_COUNT,
1357 SNB_DB_MSIX_VECTOR_SHIFT,
1358 SNB_DB_TOTAL_SHIFT);
1359}
1421 1360
1422 kfree(ndev->db_cb); 1361static void snb_deinit_isr(struct intel_ntb_dev *ndev)
1362{
1363 ndev_deinit_isr(ndev);
1423} 1364}
1424 1365
1425static ssize_t ntb_debugfs_read(struct file *filp, char __user *ubuf, 1366static int snb_setup_b2b_mw(struct intel_ntb_dev *ndev,
1426 size_t count, loff_t *offp) 1367 const struct intel_b2b_addr *addr,
1368 const struct intel_b2b_addr *peer_addr)
1427{ 1369{
1428 struct ntb_device *ndev; 1370 struct pci_dev *pdev;
1429 char *buf; 1371 void __iomem *mmio;
1430 ssize_t ret, offset, out_count; 1372 resource_size_t bar_size;
1373 phys_addr_t bar_addr;
1374 int b2b_bar;
1375 u8 bar_sz;
1376
1377 pdev = ndev_pdev(ndev);
1378 mmio = ndev->self_mmio;
1379
1380 if (ndev->b2b_idx >= ndev->mw_count) {
1381 dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
1382 b2b_bar = 0;
1383 ndev->b2b_off = 0;
1384 } else {
1385 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1386 if (b2b_bar < 0)
1387 return -EIO;
1431 1388
1432 out_count = 500; 1389 dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
1433 1390
1434 buf = kmalloc(out_count, GFP_KERNEL); 1391 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1435 if (!buf)
1436 return -ENOMEM;
1437 1392
1438 ndev = filp->private_data; 1393 dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
1439 offset = 0;
1440 offset += snprintf(buf + offset, out_count - offset,
1441 "NTB Device Information:\n");
1442 offset += snprintf(buf + offset, out_count - offset,
1443 "Connection Type - \t\t%s\n",
1444 ndev->conn_type == NTB_CONN_TRANSPARENT ?
1445 "Transparent" : (ndev->conn_type == NTB_CONN_B2B) ?
1446 "Back to back" : "Root Port");
1447 offset += snprintf(buf + offset, out_count - offset,
1448 "Device Type - \t\t\t%s\n",
1449 ndev->dev_type == NTB_DEV_USD ?
1450 "DSD/USP" : "USD/DSP");
1451 offset += snprintf(buf + offset, out_count - offset,
1452 "Max Number of Callbacks - \t%u\n",
1453 ntb_max_cbs(ndev));
1454 offset += snprintf(buf + offset, out_count - offset,
1455 "Link Status - \t\t\t%s\n",
1456 ntb_hw_link_status(ndev) ? "Up" : "Down");
1457 if (ntb_hw_link_status(ndev)) {
1458 offset += snprintf(buf + offset, out_count - offset,
1459 "Link Speed - \t\t\tPCI-E Gen %u\n",
1460 ndev->link_speed);
1461 offset += snprintf(buf + offset, out_count - offset,
1462 "Link Width - \t\t\tx%u\n",
1463 ndev->link_width);
1464 }
1465 1394
1466 if (is_ntb_xeon(ndev)) { 1395 if (b2b_mw_share && SNB_B2B_MIN_SIZE <= bar_size >> 1) {
1467 u32 status32; 1396 dev_dbg(ndev_dev(ndev),
1468 u16 status16; 1397 "b2b using first half of bar\n");
1469 int rc; 1398 ndev->b2b_off = bar_size >> 1;
1470 1399 } else if (SNB_B2B_MIN_SIZE <= bar_size) {
1471 offset += snprintf(buf + offset, out_count - offset, 1400 dev_dbg(ndev_dev(ndev),
1472 "\nNTB Device Statistics:\n"); 1401 "b2b using whole bar\n");
1473 offset += snprintf(buf + offset, out_count - offset, 1402 ndev->b2b_off = 0;
1474 "Upstream Memory Miss - \t%u\n", 1403 --ndev->mw_count;
1475 readw(ndev->reg_base + 1404 } else {
1476 SNB_USMEMMISS_OFFSET)); 1405 dev_dbg(ndev_dev(ndev),
1477 1406 "b2b bar size is too small\n");
1478 offset += snprintf(buf + offset, out_count - offset, 1407 return -EIO;
1479 "\nNTB Hardware Errors:\n"); 1408 }
1480
1481 rc = pci_read_config_word(ndev->pdev, SNB_DEVSTS_OFFSET,
1482 &status16);
1483 if (!rc)
1484 offset += snprintf(buf + offset, out_count - offset,
1485 "DEVSTS - \t%#06x\n", status16);
1486
1487 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
1488 &status16);
1489 if (!rc)
1490 offset += snprintf(buf + offset, out_count - offset,
1491 "LNKSTS - \t%#06x\n", status16);
1492
1493 rc = pci_read_config_dword(ndev->pdev, SNB_UNCERRSTS_OFFSET,
1494 &status32);
1495 if (!rc)
1496 offset += snprintf(buf + offset, out_count - offset,
1497 "UNCERRSTS - \t%#010x\n", status32);
1498
1499 rc = pci_read_config_dword(ndev->pdev, SNB_CORERRSTS_OFFSET,
1500 &status32);
1501 if (!rc)
1502 offset += snprintf(buf + offset, out_count - offset,
1503 "CORERRSTS - \t%#010x\n", status32);
1504 } 1409 }
1505 1410
1506 if (offset > out_count) 1411 /* Reset the secondary bar sizes to match the primary bar sizes,
1507 offset = out_count; 1412 * except disable or halve the size of the b2b secondary bar.
1413 *
1414 * Note: code for each specific bar size register, because the register
1415 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
1416 */
1417 pci_read_config_byte(pdev, SNB_PBAR23SZ_OFFSET, &bar_sz);
1418 dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz);
1419 if (b2b_bar == 2) {
1420 if (ndev->b2b_off)
1421 bar_sz -= 1;
1422 else
1423 bar_sz = 0;
1424 }
1425 pci_write_config_byte(pdev, SNB_SBAR23SZ_OFFSET, bar_sz);
1426 pci_read_config_byte(pdev, SNB_SBAR23SZ_OFFSET, &bar_sz);
1427 dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz);
1428
1429 if (!ndev->bar4_split) {
1430 pci_read_config_byte(pdev, SNB_PBAR45SZ_OFFSET, &bar_sz);
1431 dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz);
1432 if (b2b_bar == 4) {
1433 if (ndev->b2b_off)
1434 bar_sz -= 1;
1435 else
1436 bar_sz = 0;
1437 }
1438 pci_write_config_byte(pdev, SNB_SBAR45SZ_OFFSET, bar_sz);
1439 pci_read_config_byte(pdev, SNB_SBAR45SZ_OFFSET, &bar_sz);
1440 dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz);
1441 } else {
1442 pci_read_config_byte(pdev, SNB_PBAR4SZ_OFFSET, &bar_sz);
1443 dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz);
1444 if (b2b_bar == 4) {
1445 if (ndev->b2b_off)
1446 bar_sz -= 1;
1447 else
1448 bar_sz = 0;
1449 }
1450 pci_write_config_byte(pdev, SNB_SBAR4SZ_OFFSET, bar_sz);
1451 pci_read_config_byte(pdev, SNB_SBAR4SZ_OFFSET, &bar_sz);
1452 dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz);
1453
1454 pci_read_config_byte(pdev, SNB_PBAR5SZ_OFFSET, &bar_sz);
1455 dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz);
1456 if (b2b_bar == 5) {
1457 if (ndev->b2b_off)
1458 bar_sz -= 1;
1459 else
1460 bar_sz = 0;
1461 }
1462 pci_write_config_byte(pdev, SNB_SBAR5SZ_OFFSET, bar_sz);
1463 pci_read_config_byte(pdev, SNB_SBAR5SZ_OFFSET, &bar_sz);
1464 dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz);
1465 }
1508 1466
1509 ret = simple_read_from_buffer(ubuf, count, offp, buf, offset); 1467 /* SBAR01 hit by first part of the b2b bar */
1510 kfree(buf); 1468 if (b2b_bar == 0)
1511 return ret; 1469 bar_addr = addr->bar0_addr;
1512} 1470 else if (b2b_bar == 2)
1471 bar_addr = addr->bar2_addr64;
1472 else if (b2b_bar == 4 && !ndev->bar4_split)
1473 bar_addr = addr->bar4_addr64;
1474 else if (b2b_bar == 4)
1475 bar_addr = addr->bar4_addr32;
1476 else if (b2b_bar == 5)
1477 bar_addr = addr->bar5_addr32;
1478 else
1479 return -EIO;
1513 1480
1514static const struct file_operations ntb_debugfs_info = { 1481 dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr);
1515 .owner = THIS_MODULE, 1482 iowrite64(bar_addr, mmio + SNB_SBAR0BASE_OFFSET);
1516 .open = simple_open,
1517 .read = ntb_debugfs_read,
1518};
1519 1483
1520static void ntb_setup_debugfs(struct ntb_device *ndev) 1484 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
1521{ 1485 * The b2b bar is either disabled above, or configured half-size, and
1522 if (!debugfs_initialized()) 1486 * it starts at the PBAR xlat + offset.
1523 return; 1487 */
1524 1488
1525 if (!debugfs_dir) 1489 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1526 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 1490 iowrite64(bar_addr, mmio + SNB_SBAR23BASE_OFFSET);
1491 bar_addr = ioread64(mmio + SNB_SBAR23BASE_OFFSET);
1492 dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr);
1493
1494 if (!ndev->bar4_split) {
1495 bar_addr = addr->bar4_addr64 +
1496 (b2b_bar == 4 ? ndev->b2b_off : 0);
1497 iowrite64(bar_addr, mmio + SNB_SBAR45BASE_OFFSET);
1498 bar_addr = ioread64(mmio + SNB_SBAR45BASE_OFFSET);
1499 dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr);
1500 } else {
1501 bar_addr = addr->bar4_addr32 +
1502 (b2b_bar == 4 ? ndev->b2b_off : 0);
1503 iowrite32(bar_addr, mmio + SNB_SBAR4BASE_OFFSET);
1504 bar_addr = ioread32(mmio + SNB_SBAR4BASE_OFFSET);
1505 dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr);
1506
1507 bar_addr = addr->bar5_addr32 +
1508 (b2b_bar == 5 ? ndev->b2b_off : 0);
1509 iowrite32(bar_addr, mmio + SNB_SBAR5BASE_OFFSET);
1510 bar_addr = ioread32(mmio + SNB_SBAR5BASE_OFFSET);
1511 dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr);
1512 }
1527 1513
1528 ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev), 1514 /* setup incoming bar limits == base addrs (zero length windows) */
1529 debugfs_dir);
1530 if (ndev->debugfs_dir)
1531 ndev->debugfs_info = debugfs_create_file("info", S_IRUSR,
1532 ndev->debugfs_dir,
1533 ndev,
1534 &ntb_debugfs_info);
1535}
1536 1515
1537static void ntb_free_debugfs(struct ntb_device *ndev) 1516 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1538{ 1517 iowrite64(bar_addr, mmio + SNB_SBAR23LMT_OFFSET);
1539 debugfs_remove_recursive(ndev->debugfs_dir); 1518 bar_addr = ioread64(mmio + SNB_SBAR23LMT_OFFSET);
1519 dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr);
1540 1520
1541 if (debugfs_dir && simple_empty(debugfs_dir)) { 1521 if (!ndev->bar4_split) {
1542 debugfs_remove_recursive(debugfs_dir); 1522 bar_addr = addr->bar4_addr64 +
1543 debugfs_dir = NULL; 1523 (b2b_bar == 4 ? ndev->b2b_off : 0);
1524 iowrite64(bar_addr, mmio + SNB_SBAR45LMT_OFFSET);
1525 bar_addr = ioread64(mmio + SNB_SBAR45LMT_OFFSET);
1526 dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr);
1527 } else {
1528 bar_addr = addr->bar4_addr32 +
1529 (b2b_bar == 4 ? ndev->b2b_off : 0);
1530 iowrite32(bar_addr, mmio + SNB_SBAR4LMT_OFFSET);
1531 bar_addr = ioread32(mmio + SNB_SBAR4LMT_OFFSET);
1532 dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr);
1533
1534 bar_addr = addr->bar5_addr32 +
1535 (b2b_bar == 5 ? ndev->b2b_off : 0);
1536 iowrite32(bar_addr, mmio + SNB_SBAR5LMT_OFFSET);
1537 bar_addr = ioread32(mmio + SNB_SBAR5LMT_OFFSET);
1538 dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr);
1544 } 1539 }
1545}
1546 1540
1547static void ntb_hw_link_up(struct ntb_device *ndev) 1541 /* zero incoming translation addrs */
1548{ 1542 iowrite64(0, mmio + SNB_SBAR23XLAT_OFFSET);
1549 if (ndev->conn_type == NTB_CONN_TRANSPARENT)
1550 ntb_link_event(ndev, NTB_LINK_UP);
1551 else {
1552 u32 ntb_cntl;
1553 1543
1554 /* Let's bring the NTB link up */ 1544 if (!ndev->bar4_split) {
1555 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl); 1545 iowrite64(0, mmio + SNB_SBAR45XLAT_OFFSET);
1556 ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 1546 } else {
1557 ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 1547 iowrite32(0, mmio + SNB_SBAR4XLAT_OFFSET);
1558 ntb_cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 1548 iowrite32(0, mmio + SNB_SBAR5XLAT_OFFSET);
1559 if (ndev->split_bar) 1549 }
1560 ntb_cntl |= NTB_CNTL_P2S_BAR5_SNOOP |
1561 NTB_CNTL_S2P_BAR5_SNOOP;
1562 1550
1563 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl); 1551 /* zero outgoing translation limits (whole bar size windows) */
1552 iowrite64(0, mmio + SNB_PBAR23LMT_OFFSET);
1553 if (!ndev->bar4_split) {
1554 iowrite64(0, mmio + SNB_PBAR45LMT_OFFSET);
1555 } else {
1556 iowrite32(0, mmio + SNB_PBAR4LMT_OFFSET);
1557 iowrite32(0, mmio + SNB_PBAR5LMT_OFFSET);
1564 } 1558 }
1565}
1566 1559
1567static void ntb_hw_link_down(struct ntb_device *ndev) 1560 /* set outgoing translation offsets */
1568{ 1561 bar_addr = peer_addr->bar2_addr64;
1569 u32 ntb_cntl; 1562 iowrite64(bar_addr, mmio + SNB_PBAR23XLAT_OFFSET);
1563 bar_addr = ioread64(mmio + SNB_PBAR23XLAT_OFFSET);
1564 dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr);
1565
1566 if (!ndev->bar4_split) {
1567 bar_addr = peer_addr->bar4_addr64;
1568 iowrite64(bar_addr, mmio + SNB_PBAR45XLAT_OFFSET);
1569 bar_addr = ioread64(mmio + SNB_PBAR45XLAT_OFFSET);
1570 dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr);
1571 } else {
1572 bar_addr = peer_addr->bar4_addr32;
1573 iowrite32(bar_addr, mmio + SNB_PBAR4XLAT_OFFSET);
1574 bar_addr = ioread32(mmio + SNB_PBAR4XLAT_OFFSET);
1575 dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr);
1576
1577 bar_addr = peer_addr->bar5_addr32;
1578 iowrite32(bar_addr, mmio + SNB_PBAR5XLAT_OFFSET);
1579 bar_addr = ioread32(mmio + SNB_PBAR5XLAT_OFFSET);
1580 dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr);
1581 }
1570 1582
1571 if (ndev->conn_type == NTB_CONN_TRANSPARENT) { 1583 /* set the translation offset for b2b registers */
1572 ntb_link_event(ndev, NTB_LINK_DOWN); 1584 if (b2b_bar == 0)
1573 return; 1585 bar_addr = peer_addr->bar0_addr;
1586 else if (b2b_bar == 2)
1587 bar_addr = peer_addr->bar2_addr64;
1588 else if (b2b_bar == 4 && !ndev->bar4_split)
1589 bar_addr = peer_addr->bar4_addr64;
1590 else if (b2b_bar == 4)
1591 bar_addr = peer_addr->bar4_addr32;
1592 else if (b2b_bar == 5)
1593 bar_addr = peer_addr->bar5_addr32;
1594 else
1595 return -EIO;
1596
1597 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
1598 dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr);
1599 iowrite32(bar_addr, mmio + SNB_B2B_XLAT_OFFSETL);
1600 iowrite32(bar_addr >> 32, mmio + SNB_B2B_XLAT_OFFSETU);
1601
1602 if (b2b_bar) {
1603 /* map peer ntb mmio config space registers */
1604 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1605 SNB_B2B_MIN_SIZE);
1606 if (!ndev->peer_mmio)
1607 return -EIO;
1574 } 1608 }
1575 1609
1576 /* Bring NTB link down */ 1610 return 0;
1577 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1578 ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
1579 ntb_cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
1580 if (ndev->split_bar)
1581 ntb_cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP |
1582 NTB_CNTL_S2P_BAR5_SNOOP);
1583 ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
1584 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1585} 1611}
1586 1612
1587static void ntb_max_mw_detect(struct ntb_device *ndev) 1613static int snb_init_ntb(struct intel_ntb_dev *ndev)
1588{ 1614{
1589 if (ndev->split_bar) 1615 int rc;
1590 ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW; 1616
1617 if (ndev->bar4_split)
1618 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1591 else 1619 else
1592 ndev->limits.max_mw = SNB_MAX_MW; 1620 ndev->mw_count = SNB_MW_COUNT;
1593}
1594 1621
1595static int ntb_xeon_detect(struct ntb_device *ndev) 1622 ndev->spad_count = SNB_SPAD_COUNT;
1596{ 1623 ndev->db_count = SNB_DB_COUNT;
1597 int rc, bars_mask; 1624 ndev->db_link_mask = SNB_DB_LINK_BIT;
1598 u32 bars;
1599 u8 ppd;
1600 1625
1601 ndev->hw_type = SNB_HW; 1626 switch (ndev->ntb.topo) {
1627 case NTB_TOPO_PRI:
1628 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1629 dev_err(ndev_dev(ndev), "NTB Primary config disabled\n");
1630 return -EINVAL;
1631 }
1632 /* use half the spads for the peer */
1633 ndev->spad_count >>= 1;
1634 ndev->self_reg = &snb_pri_reg;
1635 ndev->peer_reg = &snb_sec_reg;
1636 ndev->xlat_reg = &snb_sec_xlat;
1637 break;
1602 1638
1603 rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &ppd); 1639 case NTB_TOPO_SEC:
1604 if (rc) 1640 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1605 return -EIO; 1641 dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n");
1642 return -EINVAL;
1643 }
1644 /* use half the spads for the peer */
1645 ndev->spad_count >>= 1;
1646 ndev->self_reg = &snb_sec_reg;
1647 ndev->peer_reg = &snb_pri_reg;
1648 ndev->xlat_reg = &snb_pri_xlat;
1649 break;
1606 1650
1607 if (ppd & SNB_PPD_DEV_TYPE) 1651 case NTB_TOPO_B2B_USD:
1608 ndev->dev_type = NTB_DEV_USD; 1652 case NTB_TOPO_B2B_DSD:
1609 else 1653 ndev->self_reg = &snb_pri_reg;
1610 ndev->dev_type = NTB_DEV_DSD; 1654 ndev->peer_reg = &snb_b2b_reg;
1655 ndev->xlat_reg = &snb_sec_xlat;
1611 1656
1612 ndev->split_bar = (ppd & SNB_PPD_SPLIT_BAR) ? 1 : 0; 1657 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1658 ndev->peer_reg = &snb_pri_reg;
1613 1659
1614 switch (ppd & SNB_PPD_CONN_TYPE) { 1660 if (b2b_mw_idx < 0)
1615 case NTB_CONN_B2B: 1661 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1616 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); 1662 else
1617 ndev->conn_type = NTB_CONN_B2B; 1663 ndev->b2b_idx = b2b_mw_idx;
1618 break;
1619 case NTB_CONN_RP:
1620 dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
1621 ndev->conn_type = NTB_CONN_RP;
1622 break;
1623 case NTB_CONN_TRANSPARENT:
1624 dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
1625 ndev->conn_type = NTB_CONN_TRANSPARENT;
1626 /*
1627 * This mode is default to USD/DSP. HW does not report
1628 * properly in transparent mode as it has no knowledge of
1629 * NTB. We will just force correct here.
1630 */
1631 ndev->dev_type = NTB_DEV_USD;
1632 1664
1633 /* 1665 dev_dbg(ndev_dev(ndev),
1634 * This is a way for transparent BAR to figure out if we 1666 "setting up b2b mw idx %d means %d\n",
1635 * are doing split BAR or not. There is no way for the hw 1667 b2b_mw_idx, ndev->b2b_idx);
1636 * on the transparent side to know and set the PPD. 1668
1637 */ 1669 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1638 bars_mask = pci_select_bars(ndev->pdev, IORESOURCE_MEM); 1670 dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n");
1639 bars = hweight32(bars_mask); 1671 ndev->db_count -= 1;
1640 if (bars == (HSX_SPLITBAR_MAX_MW + 1)) 1672 }
1641 ndev->split_bar = 1; 1673
1674 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1675 rc = snb_setup_b2b_mw(ndev,
1676 &snb_b2b_dsd_addr,
1677 &snb_b2b_usd_addr);
1678 } else {
1679 rc = snb_setup_b2b_mw(ndev,
1680 &snb_b2b_usd_addr,
1681 &snb_b2b_dsd_addr);
1682 }
1683 if (rc)
1684 return rc;
1685
1686 /* Enable Bus Master and Memory Space on the secondary side */
1687 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1688 ndev->self_mmio + SNB_SPCICMD_OFFSET);
1642 1689
1643 break; 1690 break;
1691
1644 default: 1692 default:
1645 dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", ppd); 1693 return -EINVAL;
1646 return -ENODEV;
1647 } 1694 }
1648 1695
1649 ntb_max_mw_detect(ndev); 1696 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1697
1698 ndev->reg->db_iowrite(ndev->db_valid_mask,
1699 ndev->self_mmio +
1700 ndev->self_reg->db_mask);
1650 1701
1651 return 0; 1702 return 0;
1652} 1703}
1653 1704
1654static int ntb_atom_detect(struct ntb_device *ndev) 1705static int snb_init_dev(struct intel_ntb_dev *ndev)
1655{ 1706{
1656 int rc; 1707 struct pci_dev *pdev;
1657 u32 ppd; 1708 u8 ppd;
1709 int rc, mem;
1710
1711 /* There is a Xeon hardware errata related to writes to SDOORBELL or
1712 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
1713 * which may hang the system. To workaround this use the second memory
1714 * window to access the interrupt and scratch pad registers on the
1715 * remote system.
1716 */
1717 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1718
1719 /* There is a hardware errata related to accessing any register in
1720 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
1721 */
1722 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1658 1723
1659 ndev->hw_type = BWD_HW; 1724 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
1660 ndev->limits.max_mw = BWD_MAX_MW; 1725 * mirrored to the remote system. Shrink the number of bits by one,
1726 * since bit 14 is the last bit.
1727 */
1728 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1729
1730 ndev->reg = &snb_reg;
1731
1732 pdev = ndev_pdev(ndev);
1661 1733
1662 rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &ppd); 1734 rc = pci_read_config_byte(pdev, SNB_PPD_OFFSET, &ppd);
1663 if (rc) 1735 if (rc)
1664 return rc; 1736 return -EIO;
1665 1737
1666 switch ((ppd & BWD_PPD_CONN_TYPE) >> 8) { 1738 ndev->ntb.topo = snb_ppd_topo(ndev, ppd);
1667 case NTB_CONN_B2B: 1739 dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
1668 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n"); 1740 ntb_topo_string(ndev->ntb.topo));
1669 ndev->conn_type = NTB_CONN_B2B; 1741 if (ndev->ntb.topo == NTB_TOPO_NONE)
1670 break;
1671 case NTB_CONN_RP:
1672 default:
1673 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
1674 return -EINVAL; 1742 return -EINVAL;
1743
1744 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1745 ndev->bar4_split = snb_ppd_bar4_split(ndev, ppd);
1746 dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n",
1747 ppd, ndev->bar4_split);
1748 } else {
1749 /* This is a way for transparent BAR to figure out if we are
1750 * doing split BAR or not. There is no way for the hw on the
1751 * transparent side to know and set the PPD.
1752 */
1753 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1754 ndev->bar4_split = hweight32(mem) ==
1755 HSX_SPLIT_BAR_MW_COUNT + 1;
1756 dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n",
1757 mem, ndev->bar4_split);
1675 } 1758 }
1676 1759
1677 if (ppd & BWD_PPD_DEV_TYPE) 1760 rc = snb_init_ntb(ndev);
1678 ndev->dev_type = NTB_DEV_DSD; 1761 if (rc)
1679 else 1762 return rc;
1680 ndev->dev_type = NTB_DEV_USD;
1681 1763
1682 return 0; 1764 return snb_init_isr(ndev);
1765}
1766
1767static void snb_deinit_dev(struct intel_ntb_dev *ndev)
1768{
1769 snb_deinit_isr(ndev);
1683} 1770}
1684 1771
1685static int ntb_device_detect(struct ntb_device *ndev) 1772static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1686{ 1773{
1687 int rc; 1774 int rc;
1688 1775
1689 if (is_ntb_xeon(ndev)) 1776 pci_set_drvdata(pdev, ndev);
1690 rc = ntb_xeon_detect(ndev); 1777
1691 else if (is_ntb_atom(ndev)) 1778 rc = pci_enable_device(pdev);
1692 rc = ntb_atom_detect(ndev); 1779 if (rc)
1693 else 1780 goto err_pci_enable;
1694 rc = -ENODEV; 1781
1782 rc = pci_request_regions(pdev, NTB_NAME);
1783 if (rc)
1784 goto err_pci_regions;
1785
1786 pci_set_master(pdev);
1787
1788 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1789 if (rc) {
1790 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1791 if (rc)
1792 goto err_dma_mask;
1793 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
1794 }
1795
1796 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1797 if (rc) {
1798 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1799 if (rc)
1800 goto err_dma_mask;
1801 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
1802 }
1695 1803
1696 dev_info(&ndev->pdev->dev, "Device Type = %s\n", 1804 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1697 ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP"); 1805 if (!ndev->self_mmio) {
1806 rc = -EIO;
1807 goto err_mmio;
1808 }
1809 ndev->peer_mmio = ndev->self_mmio;
1698 1810
1699 return 0; 1811 return 0;
1812
1813err_mmio:
1814err_dma_mask:
1815 pci_clear_master(pdev);
1816 pci_release_regions(pdev);
1817err_pci_regions:
1818 pci_disable_device(pdev);
1819err_pci_enable:
1820 pci_set_drvdata(pdev, NULL);
1821 return rc;
1700} 1822}
1701 1823
1702static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1824static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1703{ 1825{
1704 struct ntb_device *ndev; 1826 struct pci_dev *pdev = ndev_pdev(ndev);
1705 int rc, i;
1706 1827
1707 ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL); 1828 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1708 if (!ndev) 1829 pci_iounmap(pdev, ndev->peer_mmio);
1709 return -ENOMEM; 1830 pci_iounmap(pdev, ndev->self_mmio);
1710 1831
1711 ndev->pdev = pdev; 1832 pci_clear_master(pdev);
1833 pci_release_regions(pdev);
1834 pci_disable_device(pdev);
1835 pci_set_drvdata(pdev, NULL);
1836}
1712 1837
1713 ntb_set_errata_flags(ndev); 1838static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1839 struct pci_dev *pdev)
1840{
1841 ndev->ntb.pdev = pdev;
1842 ndev->ntb.topo = NTB_TOPO_NONE;
1843 ndev->ntb.ops = &intel_ntb_ops;
1714 1844
1715 ndev->link_status = NTB_LINK_DOWN; 1845 ndev->b2b_off = 0;
1716 pci_set_drvdata(pdev, ndev); 1846 ndev->b2b_idx = INT_MAX;
1717 ntb_setup_debugfs(ndev);
1718 1847
1719 rc = pci_enable_device(pdev); 1848 ndev->bar4_split = 0;
1720 if (rc)
1721 goto err;
1722 1849
1723 pci_set_master(ndev->pdev); 1850 ndev->mw_count = 0;
1851 ndev->spad_count = 0;
1852 ndev->db_count = 0;
1853 ndev->db_vec_count = 0;
1854 ndev->db_vec_shift = 0;
1724 1855
1725 rc = ntb_device_detect(ndev); 1856 ndev->ntb_ctl = 0;
1726 if (rc) 1857 ndev->lnk_sta = 0;
1727 goto err;
1728 1858
1729 ndev->mw = kcalloc(ndev->limits.max_mw, sizeof(struct ntb_mw), 1859 ndev->db_valid_mask = 0;
1730 GFP_KERNEL); 1860 ndev->db_link_mask = 0;
1731 if (!ndev->mw) { 1861 ndev->db_mask = 0;
1732 rc = -ENOMEM;
1733 goto err1;
1734 }
1735 1862
1736 if (ndev->split_bar) 1863 spin_lock_init(&ndev->db_mask_lock);
1737 rc = pci_request_selected_regions(pdev, NTB_SPLITBAR_MASK, 1864}
1738 KBUILD_MODNAME);
1739 else
1740 rc = pci_request_selected_regions(pdev, NTB_BAR_MASK,
1741 KBUILD_MODNAME);
1742 1865
1743 if (rc) 1866static int intel_ntb_pci_probe(struct pci_dev *pdev,
1744 goto err2; 1867 const struct pci_device_id *id)
1868{
1869 struct intel_ntb_dev *ndev;
1870 int rc;
1745 1871
1746 ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO); 1872 if (pdev_is_bwd(pdev)) {
1747 if (!ndev->reg_base) { 1873 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
1748 dev_warn(&pdev->dev, "Cannot remap BAR 0\n"); 1874 if (!ndev) {
1749 rc = -EIO; 1875 rc = -ENOMEM;
1750 goto err3; 1876 goto err_ndev;
1751 } 1877 }
1752 1878
1753 for (i = 0; i < ndev->limits.max_mw; i++) { 1879 ndev_init_struct(ndev, pdev);
1754 ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
1755 1880
1756 /* 1881 rc = intel_ntb_init_pci(ndev, pdev);
1757 * with the errata we need to steal last of the memory 1882 if (rc)
1758 * windows for workarounds and they point to MMIO registers. 1883 goto err_init_pci;
1759 */ 1884
1760 if ((ndev->wa_flags & WA_SNB_ERR) && 1885 rc = bwd_init_dev(ndev);
1761 (i == (ndev->limits.max_mw - 1))) { 1886 if (rc)
1762 ndev->mw[i].vbase = 1887 goto err_init_dev;
1763 ioremap_nocache(pci_resource_start(pdev,
1764 MW_TO_BAR(i)),
1765 ndev->mw[i].bar_sz);
1766 } else {
1767 ndev->mw[i].vbase =
1768 ioremap_wc(pci_resource_start(pdev,
1769 MW_TO_BAR(i)),
1770 ndev->mw[i].bar_sz);
1771 }
1772 1888
1773 dev_info(&pdev->dev, "MW %d size %llu\n", i, 1889 } else if (pdev_is_snb(pdev)) {
1774 (unsigned long long) ndev->mw[i].bar_sz); 1890 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
1775 if (!ndev->mw[i].vbase) { 1891 if (!ndev) {
1776 dev_warn(&pdev->dev, "Cannot remap BAR %d\n", 1892 rc = -ENOMEM;
1777 MW_TO_BAR(i)); 1893 goto err_ndev;
1778 rc = -EIO;
1779 goto err4;
1780 } 1894 }
1781 }
1782 1895
1783 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1896 ndev_init_struct(ndev, pdev);
1784 if (rc) {
1785 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1786 if (rc)
1787 goto err4;
1788 1897
1789 dev_warn(&pdev->dev, "Cannot DMA highmem\n"); 1898 rc = intel_ntb_init_pci(ndev, pdev);
1790 } 1899 if (rc)
1900 goto err_init_pci;
1791 1901
1792 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1902 rc = snb_init_dev(ndev);
1793 if (rc) {
1794 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1795 if (rc) 1903 if (rc)
1796 goto err4; 1904 goto err_init_dev;
1797 1905
1798 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); 1906 } else {
1907 rc = -EINVAL;
1908 goto err_ndev;
1799 } 1909 }
1800 1910
1801 rc = ntb_device_setup(ndev); 1911 ndev_reset_unsafe_flags(ndev);
1802 if (rc)
1803 goto err4;
1804 1912
1805 rc = ntb_create_callbacks(ndev); 1913 ndev->reg->poll_link(ndev);
1806 if (rc)
1807 goto err5;
1808
1809 rc = ntb_setup_interrupts(ndev);
1810 if (rc)
1811 goto err6;
1812 1914
1813 /* The scratchpad registers keep the values between rmmod/insmod, 1915 ndev_init_debugfs(ndev);
1814 * blast them now
1815 */
1816 for (i = 0; i < ndev->limits.max_spads; i++) {
1817 ntb_write_local_spad(ndev, i, 0);
1818 ntb_write_remote_spad(ndev, i, 0);
1819 }
1820 1916
1821 rc = ntb_transport_init(pdev); 1917 rc = ntb_register_device(&ndev->ntb);
1822 if (rc) 1918 if (rc)
1823 goto err7; 1919 goto err_register;
1824
1825 ntb_hw_link_up(ndev);
1826 1920
1827 return 0; 1921 return 0;
1828 1922
1829err7: 1923err_register:
1830 ntb_free_interrupts(ndev); 1924 ndev_deinit_debugfs(ndev);
1831err6: 1925 if (pdev_is_bwd(pdev))
1832 ntb_free_callbacks(ndev); 1926 bwd_deinit_dev(ndev);
1833err5: 1927 else if (pdev_is_snb(pdev))
1834 ntb_device_free(ndev); 1928 snb_deinit_dev(ndev);
1835err4: 1929err_init_dev:
1836 for (i--; i >= 0; i--) 1930 intel_ntb_deinit_pci(ndev);
1837 iounmap(ndev->mw[i].vbase); 1931err_init_pci:
1838 iounmap(ndev->reg_base);
1839err3:
1840 if (ndev->split_bar)
1841 pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK);
1842 else
1843 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1844err2:
1845 kfree(ndev->mw);
1846err1:
1847 pci_disable_device(pdev);
1848err:
1849 ntb_free_debugfs(ndev);
1850 kfree(ndev); 1932 kfree(ndev);
1851 1933err_ndev:
1852 dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
1853 return rc; 1934 return rc;
1854} 1935}
1855 1936
1856static void ntb_pci_remove(struct pci_dev *pdev) 1937static void intel_ntb_pci_remove(struct pci_dev *pdev)
1857{ 1938{
1858 struct ntb_device *ndev = pci_get_drvdata(pdev); 1939 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
1859 int i; 1940
1941 ntb_unregister_device(&ndev->ntb);
1942 ndev_deinit_debugfs(ndev);
1943 if (pdev_is_bwd(pdev))
1944 bwd_deinit_dev(ndev);
1945 else if (pdev_is_snb(pdev))
1946 snb_deinit_dev(ndev);
1947 intel_ntb_deinit_pci(ndev);
1948 kfree(ndev);
1949}
1860 1950
1861 ntb_hw_link_down(ndev); 1951static const struct intel_ntb_reg bwd_reg = {
1952 .poll_link = bwd_poll_link,
1953 .link_is_up = bwd_link_is_up,
1954 .db_ioread = bwd_db_ioread,
1955 .db_iowrite = bwd_db_iowrite,
1956 .db_size = sizeof(u64),
1957 .ntb_ctl = BWD_NTBCNTL_OFFSET,
1958 .mw_bar = {2, 4},
1959};
1862 1960
1863 ntb_transport_free(ndev->ntb_transport); 1961static const struct intel_ntb_alt_reg bwd_pri_reg = {
1962 .db_bell = BWD_PDOORBELL_OFFSET,
1963 .db_mask = BWD_PDBMSK_OFFSET,
1964 .spad = BWD_SPAD_OFFSET,
1965};
1864 1966
1865 ntb_free_interrupts(ndev); 1967static const struct intel_ntb_alt_reg bwd_b2b_reg = {
1866 ntb_free_callbacks(ndev); 1968 .db_bell = BWD_B2B_DOORBELL_OFFSET,
1867 ntb_device_free(ndev); 1969 .spad = BWD_B2B_SPAD_OFFSET,
1970};
1868 1971
1869 /* need to reset max_mw limits so we can unmap properly */ 1972static const struct intel_ntb_xlat_reg bwd_sec_xlat = {
1870 if (ndev->hw_type == SNB_HW) 1973 /* FIXME : .bar0_base = BWD_SBAR0BASE_OFFSET, */
1871 ntb_max_mw_detect(ndev); 1974 /* FIXME : .bar2_limit = BWD_SBAR2LMT_OFFSET, */
1975 .bar2_xlat = BWD_SBAR2XLAT_OFFSET,
1976};
1872 1977
1873 for (i = 0; i < ndev->limits.max_mw; i++) 1978static const struct intel_ntb_reg snb_reg = {
1874 iounmap(ndev->mw[i].vbase); 1979 .poll_link = snb_poll_link,
1980 .link_is_up = snb_link_is_up,
1981 .db_ioread = snb_db_ioread,
1982 .db_iowrite = snb_db_iowrite,
1983 .db_size = sizeof(u32),
1984 .ntb_ctl = SNB_NTBCNTL_OFFSET,
1985 .mw_bar = {2, 4, 5},
1986};
1875 1987
1876 kfree(ndev->mw); 1988static const struct intel_ntb_alt_reg snb_pri_reg = {
1877 iounmap(ndev->reg_base); 1989 .db_bell = SNB_PDOORBELL_OFFSET,
1878 if (ndev->split_bar) 1990 .db_mask = SNB_PDBMSK_OFFSET,
1879 pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK); 1991 .spad = SNB_SPAD_OFFSET,
1880 else 1992};
1881 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1882 pci_disable_device(pdev);
1883 ntb_free_debugfs(ndev);
1884 kfree(ndev);
1885}
1886 1993
1887static struct pci_driver ntb_pci_driver = { 1994static const struct intel_ntb_alt_reg snb_sec_reg = {
1995 .db_bell = SNB_SDOORBELL_OFFSET,
1996 .db_mask = SNB_SDBMSK_OFFSET,
1997 /* second half of the scratchpads */
1998 .spad = SNB_SPAD_OFFSET + (SNB_SPAD_COUNT << 1),
1999};
2000
2001static const struct intel_ntb_alt_reg snb_b2b_reg = {
2002 .db_bell = SNB_B2B_DOORBELL_OFFSET,
2003 .spad = SNB_B2B_SPAD_OFFSET,
2004};
2005
2006static const struct intel_ntb_xlat_reg snb_pri_xlat = {
2007 /* Note: no primary .bar0_base visible to the secondary side.
2008 *
2009 * The secondary side cannot get the base address stored in primary
2010 * bars. The base address is necessary to set the limit register to
2011 * any value other than zero, or unlimited.
2012 *
2013 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2014 * window by setting the limit equal to base, nor can it limit the size
2015 * of the memory window by setting the limit to base + size.
2016 */
2017 .bar2_limit = SNB_PBAR23LMT_OFFSET,
2018 .bar2_xlat = SNB_PBAR23XLAT_OFFSET,
2019};
2020
2021static const struct intel_ntb_xlat_reg snb_sec_xlat = {
2022 .bar0_base = SNB_SBAR0BASE_OFFSET,
2023 .bar2_limit = SNB_SBAR23LMT_OFFSET,
2024 .bar2_xlat = SNB_SBAR23XLAT_OFFSET,
2025};
2026
2027static const struct intel_b2b_addr snb_b2b_usd_addr = {
2028 .bar2_addr64 = SNB_B2B_BAR2_USD_ADDR64,
2029 .bar4_addr64 = SNB_B2B_BAR4_USD_ADDR64,
2030 .bar4_addr32 = SNB_B2B_BAR4_USD_ADDR32,
2031 .bar5_addr32 = SNB_B2B_BAR5_USD_ADDR32,
2032};
2033
2034static const struct intel_b2b_addr snb_b2b_dsd_addr = {
2035 .bar2_addr64 = SNB_B2B_BAR2_DSD_ADDR64,
2036 .bar4_addr64 = SNB_B2B_BAR4_DSD_ADDR64,
2037 .bar4_addr32 = SNB_B2B_BAR4_DSD_ADDR32,
2038 .bar5_addr32 = SNB_B2B_BAR5_DSD_ADDR32,
2039};
2040
2041/* operations for primary side of local ntb */
2042static const struct ntb_dev_ops intel_ntb_ops = {
2043 .mw_count = intel_ntb_mw_count,
2044 .mw_get_range = intel_ntb_mw_get_range,
2045 .mw_set_trans = intel_ntb_mw_set_trans,
2046 .link_is_up = intel_ntb_link_is_up,
2047 .link_enable = intel_ntb_link_enable,
2048 .link_disable = intel_ntb_link_disable,
2049 .db_is_unsafe = intel_ntb_db_is_unsafe,
2050 .db_valid_mask = intel_ntb_db_valid_mask,
2051 .db_vector_count = intel_ntb_db_vector_count,
2052 .db_vector_mask = intel_ntb_db_vector_mask,
2053 .db_read = intel_ntb_db_read,
2054 .db_clear = intel_ntb_db_clear,
2055 .db_set_mask = intel_ntb_db_set_mask,
2056 .db_clear_mask = intel_ntb_db_clear_mask,
2057 .peer_db_addr = intel_ntb_peer_db_addr,
2058 .peer_db_set = intel_ntb_peer_db_set,
2059 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2060 .spad_count = intel_ntb_spad_count,
2061 .spad_read = intel_ntb_spad_read,
2062 .spad_write = intel_ntb_spad_write,
2063 .peer_spad_addr = intel_ntb_peer_spad_addr,
2064 .peer_spad_read = intel_ntb_peer_spad_read,
2065 .peer_spad_write = intel_ntb_peer_spad_write,
2066};
2067
2068static const struct file_operations intel_ntb_debugfs_info = {
2069 .owner = THIS_MODULE,
2070 .open = simple_open,
2071 .read = ndev_debugfs_read,
2072};
2073
2074static const struct pci_device_id intel_ntb_pci_tbl[] = {
2075 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
2076 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2077 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2078 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2079 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2080 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2081 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2082 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2083 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2084 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2085 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2086 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2087 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2088 {0}
2089};
2090MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2091
2092static struct pci_driver intel_ntb_pci_driver = {
1888 .name = KBUILD_MODNAME, 2093 .name = KBUILD_MODNAME,
1889 .id_table = ntb_pci_tbl, 2094 .id_table = intel_ntb_pci_tbl,
1890 .probe = ntb_pci_probe, 2095 .probe = intel_ntb_pci_probe,
1891 .remove = ntb_pci_remove, 2096 .remove = intel_ntb_pci_remove,
1892}; 2097};
1893 2098
1894module_pci_driver(ntb_pci_driver); 2099static int __init intel_ntb_pci_driver_init(void)
2100{
2101 if (debugfs_initialized())
2102 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2103
2104 return pci_register_driver(&intel_ntb_pci_driver);
2105}
2106module_init(intel_ntb_pci_driver_init);
2107
2108static void __exit intel_ntb_pci_driver_exit(void)
2109{
2110 pci_unregister_driver(&intel_ntb_pci_driver);
2111
2112 debugfs_remove_recursive(debugfs_dir);
2113}
2114module_exit(intel_ntb_pci_driver_exit);
2115
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h
index 935610454f70..fec689dc95cf 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -5,6 +5,7 @@
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
13 * BSD LICENSE 14 * BSD LICENSE
14 * 15 *
15 * Copyright(c) 2012 Intel Corporation. All rights reserved. 16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
16 * 18 *
17 * Redistribution and use in source and binary forms, with or without 19 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions 20 * modification, are permitted provided that the following conditions
@@ -45,341 +47,296 @@
45 * Contact Information: 47 * Contact Information:
46 * Jon Mason <jon.mason@intel.com> 48 * Jon Mason <jon.mason@intel.com>
47 */ 49 */
48#include <linux/ntb_transport.h>
49
50#define NTB_LINK_STATUS_ACTIVE 0x2000
51#define NTB_LINK_SPEED_MASK 0x000f
52#define NTB_LINK_WIDTH_MASK 0x03f0
53
54#define SNB_MSIX_CNT 4
55#define SNB_MAX_B2B_SPADS 16
56#define SNB_MAX_COMPAT_SPADS 16
57/* Reserve the uppermost bit for link interrupt */
58#define SNB_MAX_DB_BITS 15
59#define SNB_LINK_DB 15
60#define SNB_DB_BITS_PER_VEC 5
61#define HSX_SPLITBAR_MAX_MW 3
62#define SNB_MAX_MW 2
63#define SNB_ERRATA_MAX_MW 1
64
65#define SNB_DB_HW_LINK 0x8000
66
67#define SNB_UNCERRSTS_OFFSET 0x014C
68#define SNB_CORERRSTS_OFFSET 0x0158
69#define SNB_LINK_STATUS_OFFSET 0x01A2
70#define SNB_PCICMD_OFFSET 0x0504
71#define SNB_DEVCTRL_OFFSET 0x0598
72#define SNB_DEVSTS_OFFSET 0x059A
73#define SNB_SLINK_STATUS_OFFSET 0x05A2
74
75#define SNB_PBAR2LMT_OFFSET 0x0000
76#define SNB_PBAR4LMT_OFFSET 0x0008
77#define SNB_PBAR5LMT_OFFSET 0x000C
78#define SNB_PBAR2XLAT_OFFSET 0x0010
79#define SNB_PBAR4XLAT_OFFSET 0x0018
80#define SNB_PBAR5XLAT_OFFSET 0x001C
81#define SNB_SBAR2LMT_OFFSET 0x0020
82#define SNB_SBAR4LMT_OFFSET 0x0028
83#define SNB_SBAR5LMT_OFFSET 0x002C
84#define SNB_SBAR2XLAT_OFFSET 0x0030
85#define SNB_SBAR4XLAT_OFFSET 0x0038
86#define SNB_SBAR5XLAT_OFFSET 0x003C
87#define SNB_SBAR0BASE_OFFSET 0x0040
88#define SNB_SBAR2BASE_OFFSET 0x0048
89#define SNB_SBAR4BASE_OFFSET 0x0050
90#define SNB_SBAR5BASE_OFFSET 0x0054
91#define SNB_NTBCNTL_OFFSET 0x0058
92#define SNB_SBDF_OFFSET 0x005C
93#define SNB_PDOORBELL_OFFSET 0x0060
94#define SNB_PDBMSK_OFFSET 0x0062
95#define SNB_SDOORBELL_OFFSET 0x0064
96#define SNB_SDBMSK_OFFSET 0x0066
97#define SNB_USMEMMISS_OFFSET 0x0070
98#define SNB_SPAD_OFFSET 0x0080
99#define SNB_SPADSEMA4_OFFSET 0x00c0
100#define SNB_WCCNTRL_OFFSET 0x00e0
101#define SNB_B2B_SPAD_OFFSET 0x0100
102#define SNB_B2B_DOORBELL_OFFSET 0x0140
103#define SNB_B2B_XLAT_OFFSETL 0x0144
104#define SNB_B2B_XLAT_OFFSETU 0x0148
105 50
106/* 51#ifndef NTB_HW_INTEL_H
107 * The addresses are setup so the 32bit BARs can function. Thus 52#define NTB_HW_INTEL_H
108 * the addresses are all in 32bit space 53
109 */ 54#include <linux/ntb.h>
110#define SNB_MBAR01_USD_ADDR 0x000000002100000CULL 55#include <linux/pci.h>
111#define SNB_MBAR23_USD_ADDR 0x000000004100000CULL 56
112#define SNB_MBAR4_USD_ADDR 0x000000008100000CULL 57#define PCI_DEVICE_ID_INTEL_NTB_B2B_JSF 0x3725
113#define SNB_MBAR5_USD_ADDR 0x00000000A100000CULL 58#define PCI_DEVICE_ID_INTEL_NTB_PS_JSF 0x3726
114#define SNB_MBAR01_DSD_ADDR 0x000000002000000CULL 59#define PCI_DEVICE_ID_INTEL_NTB_SS_JSF 0x3727
115#define SNB_MBAR23_DSD_ADDR 0x000000004000000CULL 60#define PCI_DEVICE_ID_INTEL_NTB_B2B_SNB 0x3C0D
116#define SNB_MBAR4_DSD_ADDR 0x000000008000000CULL 61#define PCI_DEVICE_ID_INTEL_NTB_PS_SNB 0x3C0E
117#define SNB_MBAR5_DSD_ADDR 0x00000000A000000CULL 62#define PCI_DEVICE_ID_INTEL_NTB_SS_SNB 0x3C0F
118 63#define PCI_DEVICE_ID_INTEL_NTB_B2B_IVT 0x0E0D
119#define BWD_MSIX_CNT 34 64#define PCI_DEVICE_ID_INTEL_NTB_PS_IVT 0x0E0E
120#define BWD_MAX_SPADS 16 65#define PCI_DEVICE_ID_INTEL_NTB_SS_IVT 0x0E0F
121#define BWD_MAX_DB_BITS 34 66#define PCI_DEVICE_ID_INTEL_NTB_B2B_HSX 0x2F0D
122#define BWD_DB_BITS_PER_VEC 1 67#define PCI_DEVICE_ID_INTEL_NTB_PS_HSX 0x2F0E
123#define BWD_MAX_MW 2 68#define PCI_DEVICE_ID_INTEL_NTB_SS_HSX 0x2F0F
124 69#define PCI_DEVICE_ID_INTEL_NTB_B2B_BWD 0x0C4E
125#define BWD_PCICMD_OFFSET 0xb004 70
126#define BWD_MBAR23_OFFSET 0xb018 71/* SNB hardware (and JSF, IVT, HSX) */
127#define BWD_MBAR45_OFFSET 0xb020 72
128#define BWD_DEVCTRL_OFFSET 0xb048 73#define SNB_PBAR23LMT_OFFSET 0x0000
129#define BWD_LINK_STATUS_OFFSET 0xb052 74#define SNB_PBAR45LMT_OFFSET 0x0008
130#define BWD_ERRCORSTS_OFFSET 0xb110 75#define SNB_PBAR4LMT_OFFSET 0x0008
131 76#define SNB_PBAR5LMT_OFFSET 0x000c
132#define BWD_SBAR2XLAT_OFFSET 0x0008 77#define SNB_PBAR23XLAT_OFFSET 0x0010
133#define BWD_SBAR4XLAT_OFFSET 0x0010 78#define SNB_PBAR45XLAT_OFFSET 0x0018
134#define BWD_PDOORBELL_OFFSET 0x0020 79#define SNB_PBAR4XLAT_OFFSET 0x0018
135#define BWD_PDBMSK_OFFSET 0x0028 80#define SNB_PBAR5XLAT_OFFSET 0x001c
136#define BWD_NTBCNTL_OFFSET 0x0060 81#define SNB_SBAR23LMT_OFFSET 0x0020
137#define BWD_EBDF_OFFSET 0x0064 82#define SNB_SBAR45LMT_OFFSET 0x0028
138#define BWD_SPAD_OFFSET 0x0080 83#define SNB_SBAR4LMT_OFFSET 0x0028
139#define BWD_SPADSEMA_OFFSET 0x00c0 84#define SNB_SBAR5LMT_OFFSET 0x002c
140#define BWD_STKYSPAD_OFFSET 0x00c4 85#define SNB_SBAR23XLAT_OFFSET 0x0030
141#define BWD_PBAR2XLAT_OFFSET 0x8008 86#define SNB_SBAR45XLAT_OFFSET 0x0038
142#define BWD_PBAR4XLAT_OFFSET 0x8010 87#define SNB_SBAR4XLAT_OFFSET 0x0038
143#define BWD_B2B_DOORBELL_OFFSET 0x8020 88#define SNB_SBAR5XLAT_OFFSET 0x003c
144#define BWD_B2B_SPAD_OFFSET 0x8080 89#define SNB_SBAR0BASE_OFFSET 0x0040
145#define BWD_B2B_SPADSEMA_OFFSET 0x80c0 90#define SNB_SBAR23BASE_OFFSET 0x0048
146#define BWD_B2B_STKYSPAD_OFFSET 0x80c4 91#define SNB_SBAR45BASE_OFFSET 0x0050
147 92#define SNB_SBAR4BASE_OFFSET 0x0050
148#define BWD_MODPHY_PCSREG4 0x1c004 93#define SNB_SBAR5BASE_OFFSET 0x0054
149#define BWD_MODPHY_PCSREG6 0x1c006 94#define SNB_SBDF_OFFSET 0x005c
150 95#define SNB_NTBCNTL_OFFSET 0x0058
151#define BWD_IP_BASE 0xC000 96#define SNB_PDOORBELL_OFFSET 0x0060
152#define BWD_DESKEWSTS_OFFSET (BWD_IP_BASE + 0x3024) 97#define SNB_PDBMSK_OFFSET 0x0062
153#define BWD_LTSSMERRSTS0_OFFSET (BWD_IP_BASE + 0x3180) 98#define SNB_SDOORBELL_OFFSET 0x0064
99#define SNB_SDBMSK_OFFSET 0x0066
100#define SNB_USMEMMISS_OFFSET 0x0070
101#define SNB_SPAD_OFFSET 0x0080
102#define SNB_PBAR23SZ_OFFSET 0x00d0
103#define SNB_PBAR45SZ_OFFSET 0x00d1
104#define SNB_PBAR4SZ_OFFSET 0x00d1
105#define SNB_SBAR23SZ_OFFSET 0x00d2
106#define SNB_SBAR45SZ_OFFSET 0x00d3
107#define SNB_SBAR4SZ_OFFSET 0x00d3
108#define SNB_PPD_OFFSET 0x00d4
109#define SNB_PBAR5SZ_OFFSET 0x00d5
110#define SNB_SBAR5SZ_OFFSET 0x00d6
111#define SNB_WCCNTRL_OFFSET 0x00e0
112#define SNB_UNCERRSTS_OFFSET 0x014c
113#define SNB_CORERRSTS_OFFSET 0x0158
114#define SNB_LINK_STATUS_OFFSET 0x01a2
115#define SNB_SPCICMD_OFFSET 0x0504
116#define SNB_DEVCTRL_OFFSET 0x0598
117#define SNB_DEVSTS_OFFSET 0x059a
118#define SNB_SLINK_STATUS_OFFSET 0x05a2
119#define SNB_B2B_SPAD_OFFSET 0x0100
120#define SNB_B2B_DOORBELL_OFFSET 0x0140
121#define SNB_B2B_XLAT_OFFSETL 0x0144
122#define SNB_B2B_XLAT_OFFSETU 0x0148
123#define SNB_PPD_CONN_MASK 0x03
124#define SNB_PPD_CONN_TRANSPARENT 0x00
125#define SNB_PPD_CONN_B2B 0x01
126#define SNB_PPD_CONN_RP 0x02
127#define SNB_PPD_DEV_MASK 0x10
128#define SNB_PPD_DEV_USD 0x00
129#define SNB_PPD_DEV_DSD 0x10
130#define SNB_PPD_SPLIT_BAR_MASK 0x40
131
132#define SNB_PPD_TOPO_MASK (SNB_PPD_CONN_MASK | SNB_PPD_DEV_MASK)
133#define SNB_PPD_TOPO_PRI_USD (SNB_PPD_CONN_RP | SNB_PPD_DEV_USD)
134#define SNB_PPD_TOPO_PRI_DSD (SNB_PPD_CONN_RP | SNB_PPD_DEV_DSD)
135#define SNB_PPD_TOPO_SEC_USD (SNB_PPD_CONN_TRANSPARENT | SNB_PPD_DEV_USD)
136#define SNB_PPD_TOPO_SEC_DSD (SNB_PPD_CONN_TRANSPARENT | SNB_PPD_DEV_DSD)
137#define SNB_PPD_TOPO_B2B_USD (SNB_PPD_CONN_B2B | SNB_PPD_DEV_USD)
138#define SNB_PPD_TOPO_B2B_DSD (SNB_PPD_CONN_B2B | SNB_PPD_DEV_DSD)
139
140#define SNB_MW_COUNT 2
141#define HSX_SPLIT_BAR_MW_COUNT 3
142#define SNB_DB_COUNT 15
143#define SNB_DB_LINK 15
144#define SNB_DB_LINK_BIT BIT_ULL(SNB_DB_LINK)
145#define SNB_DB_MSIX_VECTOR_COUNT 4
146#define SNB_DB_MSIX_VECTOR_SHIFT 5
147#define SNB_DB_TOTAL_SHIFT 16
148#define SNB_SPAD_COUNT 16
149
150/* BWD hardware */
151
152#define BWD_SBAR2XLAT_OFFSET 0x0008
153#define BWD_PDOORBELL_OFFSET 0x0020
154#define BWD_PDBMSK_OFFSET 0x0028
155#define BWD_NTBCNTL_OFFSET 0x0060
156#define BWD_SPAD_OFFSET 0x0080
157#define BWD_PPD_OFFSET 0x00d4
158#define BWD_PBAR2XLAT_OFFSET 0x8008
159#define BWD_B2B_DOORBELL_OFFSET 0x8020
160#define BWD_B2B_SPAD_OFFSET 0x8080
161#define BWD_SPCICMD_OFFSET 0xb004
162#define BWD_LINK_STATUS_OFFSET 0xb052
163#define BWD_ERRCORSTS_OFFSET 0xb110
164#define BWD_IP_BASE 0xc000
165#define BWD_DESKEWSTS_OFFSET (BWD_IP_BASE + 0x3024)
166#define BWD_LTSSMERRSTS0_OFFSET (BWD_IP_BASE + 0x3180)
154#define BWD_LTSSMSTATEJMP_OFFSET (BWD_IP_BASE + 0x3040) 167#define BWD_LTSSMSTATEJMP_OFFSET (BWD_IP_BASE + 0x3040)
155#define BWD_IBSTERRRCRVSTS0_OFFSET (BWD_IP_BASE + 0x3324) 168#define BWD_IBSTERRRCRVSTS0_OFFSET (BWD_IP_BASE + 0x3324)
169#define BWD_MODPHY_PCSREG4 0x1c004
170#define BWD_MODPHY_PCSREG6 0x1c006
171
172#define BWD_PPD_INIT_LINK 0x0008
173#define BWD_PPD_CONN_MASK 0x0300
174#define BWD_PPD_CONN_TRANSPARENT 0x0000
175#define BWD_PPD_CONN_B2B 0x0100
176#define BWD_PPD_CONN_RP 0x0200
177#define BWD_PPD_DEV_MASK 0x1000
178#define BWD_PPD_DEV_USD 0x0000
179#define BWD_PPD_DEV_DSD 0x1000
180#define BWD_PPD_TOPO_MASK (BWD_PPD_CONN_MASK | BWD_PPD_DEV_MASK)
181#define BWD_PPD_TOPO_PRI_USD (BWD_PPD_CONN_TRANSPARENT | BWD_PPD_DEV_USD)
182#define BWD_PPD_TOPO_PRI_DSD (BWD_PPD_CONN_TRANSPARENT | BWD_PPD_DEV_DSD)
183#define BWD_PPD_TOPO_SEC_USD (BWD_PPD_CONN_RP | BWD_PPD_DEV_USD)
184#define BWD_PPD_TOPO_SEC_DSD (BWD_PPD_CONN_RP | BWD_PPD_DEV_DSD)
185#define BWD_PPD_TOPO_B2B_USD (BWD_PPD_CONN_B2B | BWD_PPD_DEV_USD)
186#define BWD_PPD_TOPO_B2B_DSD (BWD_PPD_CONN_B2B | BWD_PPD_DEV_DSD)
187
188#define BWD_MW_COUNT 2
189#define BWD_DB_COUNT 34
190#define BWD_DB_VALID_MASK (BIT_ULL(BWD_DB_COUNT) - 1)
191#define BWD_DB_MSIX_VECTOR_COUNT 34
192#define BWD_DB_MSIX_VECTOR_SHIFT 1
193#define BWD_DB_TOTAL_SHIFT 34
194#define BWD_SPAD_COUNT 16
195
196#define BWD_NTB_CTL_DOWN_BIT BIT(16)
197#define BWD_NTB_CTL_ACTIVE(x) !(x & BWD_NTB_CTL_DOWN_BIT)
198
199#define BWD_DESKEWSTS_DBERR BIT(15)
200#define BWD_LTSSMERRSTS0_UNEXPECTEDEI BIT(20)
201#define BWD_LTSSMSTATEJMP_FORCEDETECT BIT(2)
202#define BWD_IBIST_ERR_OFLOW 0x7FFF7FFF
203
204#define BWD_LINK_HB_TIMEOUT msecs_to_jiffies(1000)
205#define BWD_LINK_RECOVERY_TIME msecs_to_jiffies(500)
206
207/* Ntb control and link status */
208
209#define NTB_CTL_CFG_LOCK BIT(0)
210#define NTB_CTL_DISABLE BIT(1)
211#define NTB_CTL_S2P_BAR2_SNOOP BIT(2)
212#define NTB_CTL_P2S_BAR2_SNOOP BIT(4)
213#define NTB_CTL_S2P_BAR4_SNOOP BIT(6)
214#define NTB_CTL_P2S_BAR4_SNOOP BIT(8)
215#define NTB_CTL_S2P_BAR5_SNOOP BIT(12)
216#define NTB_CTL_P2S_BAR5_SNOOP BIT(14)
217
218#define NTB_LNK_STA_ACTIVE_BIT 0x2000
219#define NTB_LNK_STA_SPEED_MASK 0x000f
220#define NTB_LNK_STA_WIDTH_MASK 0x03f0
221#define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LNK_STA_ACTIVE_BIT))
222#define NTB_LNK_STA_SPEED(x) ((x) & NTB_LNK_STA_SPEED_MASK)
223#define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 4)
224
225/* Use the following addresses for translation between b2b ntb devices in case
226 * the hardware default values are not reliable. */
227#define SNB_B2B_BAR0_USD_ADDR 0x1000000000000000ull
228#define SNB_B2B_BAR2_USD_ADDR64 0x2000000000000000ull
229#define SNB_B2B_BAR4_USD_ADDR64 0x4000000000000000ull
230#define SNB_B2B_BAR4_USD_ADDR32 0x20000000u
231#define SNB_B2B_BAR5_USD_ADDR32 0x40000000u
232#define SNB_B2B_BAR0_DSD_ADDR 0x9000000000000000ull
233#define SNB_B2B_BAR2_DSD_ADDR64 0xa000000000000000ull
234#define SNB_B2B_BAR4_DSD_ADDR64 0xc000000000000000ull
235#define SNB_B2B_BAR4_DSD_ADDR32 0xa0000000u
236#define SNB_B2B_BAR5_DSD_ADDR32 0xc0000000u
237
238/* The peer ntb secondary config space is 32KB fixed size */
239#define SNB_B2B_MIN_SIZE 0x8000
240
241/* flags to indicate hardware errata */
242#define NTB_HWERR_SDOORBELL_LOCKUP BIT_ULL(0)
243#define NTB_HWERR_SB01BASE_LOCKUP BIT_ULL(1)
244#define NTB_HWERR_B2BDOORBELL_BIT14 BIT_ULL(2)
245
246/* flags to indicate unsafe api */
247#define NTB_UNSAFE_DB BIT_ULL(0)
248#define NTB_UNSAFE_SPAD BIT_ULL(1)
249
250struct intel_ntb_dev;
251
252struct intel_ntb_reg {
253 int (*poll_link)(struct intel_ntb_dev *ndev);
254 int (*link_is_up)(struct intel_ntb_dev *ndev);
255 u64 (*db_ioread)(void __iomem *mmio);
256 void (*db_iowrite)(u64 db_bits, void __iomem *mmio);
257 unsigned long ntb_ctl;
258 resource_size_t db_size;
259 int mw_bar[];
260};
156 261
157#define BWD_DESKEWSTS_DBERR (1 << 15) 262struct intel_ntb_alt_reg {
158#define BWD_LTSSMERRSTS0_UNEXPECTEDEI (1 << 20) 263 unsigned long db_bell;
159#define BWD_LTSSMSTATEJMP_FORCEDETECT (1 << 2) 264 unsigned long db_mask;
160#define BWD_IBIST_ERR_OFLOW 0x7FFF7FFF 265 unsigned long spad;
161 266};
162#define NTB_CNTL_CFG_LOCK (1 << 0)
163#define NTB_CNTL_LINK_DISABLE (1 << 1)
164#define NTB_CNTL_S2P_BAR23_SNOOP (1 << 2)
165#define NTB_CNTL_P2S_BAR23_SNOOP (1 << 4)
166#define NTB_CNTL_S2P_BAR4_SNOOP (1 << 6)
167#define NTB_CNTL_P2S_BAR4_SNOOP (1 << 8)
168#define NTB_CNTL_S2P_BAR5_SNOOP (1 << 12)
169#define NTB_CNTL_P2S_BAR5_SNOOP (1 << 14)
170#define BWD_CNTL_LINK_DOWN (1 << 16)
171
172#define NTB_PPD_OFFSET 0x00D4
173#define SNB_PPD_CONN_TYPE 0x0003
174#define SNB_PPD_DEV_TYPE 0x0010
175#define SNB_PPD_SPLIT_BAR (1 << 6)
176#define BWD_PPD_INIT_LINK 0x0008
177#define BWD_PPD_CONN_TYPE 0x0300
178#define BWD_PPD_DEV_TYPE 0x1000
179#define PCI_DEVICE_ID_INTEL_NTB_B2B_JSF 0x3725
180#define PCI_DEVICE_ID_INTEL_NTB_PS_JSF 0x3726
181#define PCI_DEVICE_ID_INTEL_NTB_SS_JSF 0x3727
182#define PCI_DEVICE_ID_INTEL_NTB_B2B_SNB 0x3C0D
183#define PCI_DEVICE_ID_INTEL_NTB_PS_SNB 0x3C0E
184#define PCI_DEVICE_ID_INTEL_NTB_SS_SNB 0x3C0F
185#define PCI_DEVICE_ID_INTEL_NTB_B2B_IVT 0x0E0D
186#define PCI_DEVICE_ID_INTEL_NTB_PS_IVT 0x0E0E
187#define PCI_DEVICE_ID_INTEL_NTB_SS_IVT 0x0E0F
188#define PCI_DEVICE_ID_INTEL_NTB_B2B_HSX 0x2F0D
189#define PCI_DEVICE_ID_INTEL_NTB_PS_HSX 0x2F0E
190#define PCI_DEVICE_ID_INTEL_NTB_SS_HSX 0x2F0F
191#define PCI_DEVICE_ID_INTEL_NTB_B2B_BWD 0x0C4E
192
193#ifndef readq
194static inline u64 readq(void __iomem *addr)
195{
196 return readl(addr) | (((u64) readl(addr + 4)) << 32LL);
197}
198#endif
199
200#ifndef writeq
201static inline void writeq(u64 val, void __iomem *addr)
202{
203 writel(val & 0xffffffff, addr);
204 writel(val >> 32, addr + 4);
205}
206#endif
207 267
208#define NTB_BAR_MMIO 0 268struct intel_ntb_xlat_reg {
209#define NTB_BAR_23 2 269 unsigned long bar0_base;
210#define NTB_BAR_4 4 270 unsigned long bar2_xlat;
211#define NTB_BAR_5 5 271 unsigned long bar2_limit;
212
213#define NTB_BAR_MASK ((1 << NTB_BAR_MMIO) | (1 << NTB_BAR_23) |\
214 (1 << NTB_BAR_4))
215#define NTB_SPLITBAR_MASK ((1 << NTB_BAR_MMIO) | (1 << NTB_BAR_23) |\
216 (1 << NTB_BAR_4) | (1 << NTB_BAR_5))
217
218#define NTB_HB_TIMEOUT msecs_to_jiffies(1000)
219
220enum ntb_hw_event {
221 NTB_EVENT_SW_EVENT0 = 0,
222 NTB_EVENT_SW_EVENT1,
223 NTB_EVENT_SW_EVENT2,
224 NTB_EVENT_HW_ERROR,
225 NTB_EVENT_HW_LINK_UP,
226 NTB_EVENT_HW_LINK_DOWN,
227}; 272};
228 273
229struct ntb_mw { 274struct intel_b2b_addr {
230 dma_addr_t phys_addr; 275 phys_addr_t bar0_addr;
231 void __iomem *vbase; 276 phys_addr_t bar2_addr64;
232 resource_size_t bar_sz; 277 phys_addr_t bar4_addr64;
278 phys_addr_t bar4_addr32;
279 phys_addr_t bar5_addr32;
233}; 280};
234 281
235struct ntb_db_cb { 282struct intel_ntb_vec {
236 int (*callback)(void *data, int db_num); 283 struct intel_ntb_dev *ndev;
237 unsigned int db_num; 284 int num;
238 void *data;
239 struct ntb_device *ndev;
240 struct tasklet_struct irq_work;
241}; 285};
242 286
243#define WA_SNB_ERR 0x00000001 287struct intel_ntb_dev {
244 288 struct ntb_dev ntb;
245struct ntb_device { 289
246 struct pci_dev *pdev; 290 /* offset of peer bar0 in b2b bar */
247 struct msix_entry *msix_entries; 291 unsigned long b2b_off;
248 void __iomem *reg_base; 292 /* mw idx used to access peer bar0 */
249 struct ntb_mw *mw; 293 unsigned int b2b_idx;
250 struct { 294
251 unsigned char max_mw; 295 /* BAR45 is split into BAR4 and BAR5 */
252 unsigned char max_spads; 296 bool bar4_split;
253 unsigned char max_db_bits; 297
254 unsigned char msix_cnt; 298 u32 ntb_ctl;
255 } limits; 299 u32 lnk_sta;
256 struct { 300
257 void __iomem *ldb; 301 unsigned char mw_count;
258 void __iomem *ldb_mask; 302 unsigned char spad_count;
259 void __iomem *rdb; 303 unsigned char db_count;
260 void __iomem *bar2_xlat; 304 unsigned char db_vec_count;
261 void __iomem *bar4_xlat; 305 unsigned char db_vec_shift;
262 void __iomem *bar5_xlat; 306
263 void __iomem *spad_write; 307 u64 db_valid_mask;
264 void __iomem *spad_read; 308 u64 db_link_mask;
265 void __iomem *lnk_cntl; 309 u64 db_mask;
266 void __iomem *lnk_stat; 310
267 void __iomem *spci_cmd; 311 /* synchronize rmw access of db_mask and hw reg */
268 } reg_ofs; 312 spinlock_t db_mask_lock;
269 struct ntb_transport *ntb_transport; 313
270 void (*event_cb)(void *handle, enum ntb_hw_event event); 314 struct msix_entry *msix;
271 315 struct intel_ntb_vec *vec;
272 struct ntb_db_cb *db_cb; 316
273 unsigned char hw_type; 317 const struct intel_ntb_reg *reg;
274 unsigned char conn_type; 318 const struct intel_ntb_alt_reg *self_reg;
275 unsigned char dev_type; 319 const struct intel_ntb_alt_reg *peer_reg;
276 unsigned char num_msix; 320 const struct intel_ntb_xlat_reg *xlat_reg;
277 unsigned char bits_per_vector; 321 void __iomem *self_mmio;
278 unsigned char max_cbs; 322 void __iomem *peer_mmio;
279 unsigned char link_width; 323 phys_addr_t peer_addr;
280 unsigned char link_speed; 324
281 unsigned char link_status; 325 unsigned long last_ts;
282 unsigned char split_bar; 326 struct delayed_work hb_timer;
283 327
284 struct delayed_work hb_timer; 328 unsigned long hwerr_flags;
285 unsigned long last_ts; 329 unsigned long unsafe_flags;
286 330 unsigned long unsafe_flags_ignore;
287 struct delayed_work lr_timer; 331
288 332 struct dentry *debugfs_dir;
289 struct dentry *debugfs_dir; 333 struct dentry *debugfs_info;
290 struct dentry *debugfs_info;
291
292 unsigned int wa_flags;
293}; 334};
294 335
295/** 336#define ndev_pdev(ndev) ((ndev)->ntb.pdev)
296 * ntb_max_cbs() - return the max callbacks 337#define ndev_name(ndev) pci_name(ndev_pdev(ndev))
297 * @ndev: pointer to ntb_device instance 338#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
298 * 339#define ntb_ndev(ntb) container_of(ntb, struct intel_ntb_dev, ntb)
299 * Given the ntb pointer, return the maximum number of callbacks 340#define hb_ndev(work) container_of(work, struct intel_ntb_dev, hb_timer.work)
300 * 341
301 * RETURNS: the maximum number of callbacks 342#endif
302 */
303static inline unsigned char ntb_max_cbs(struct ntb_device *ndev)
304{
305 return ndev->max_cbs;
306}
307
308/**
309 * ntb_max_mw() - return the max number of memory windows
310 * @ndev: pointer to ntb_device instance
311 *
312 * Given the ntb pointer, return the maximum number of memory windows
313 *
314 * RETURNS: the maximum number of memory windows
315 */
316static inline unsigned char ntb_max_mw(struct ntb_device *ndev)
317{
318 return ndev->limits.max_mw;
319}
320
321/**
322 * ntb_hw_link_status() - return the hardware link status
323 * @ndev: pointer to ntb_device instance
324 *
325 * Returns true if the hardware is connected to the remote system
326 *
327 * RETURNS: true or false based on the hardware link state
328 */
329static inline bool ntb_hw_link_status(struct ntb_device *ndev)
330{
331 return ndev->link_status == NTB_LINK_UP;
332}
333
334/**
335 * ntb_query_pdev() - return the pci_dev pointer
336 * @ndev: pointer to ntb_device instance
337 *
338 * Given the ntb pointer, return the pci_dev pointer for the NTB hardware device
339 *
340 * RETURNS: a pointer to the ntb pci_dev
341 */
342static inline struct pci_dev *ntb_query_pdev(struct ntb_device *ndev)
343{
344 return ndev->pdev;
345}
346
347/**
348 * ntb_query_debugfs() - return the debugfs pointer
349 * @ndev: pointer to ntb_device instance
350 *
351 * Given the ntb pointer, return the debugfs directory pointer for the NTB
352 * hardware device
353 *
354 * RETURNS: a pointer to the debugfs directory
355 */
356static inline struct dentry *ntb_query_debugfs(struct ntb_device *ndev)
357{
358 return ndev->debugfs_dir;
359}
360
361struct ntb_device *ntb_register_transport(struct pci_dev *pdev,
362 void *transport);
363void ntb_unregister_transport(struct ntb_device *ndev);
364void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr);
365int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
366 void *data, int (*db_cb_func)(void *data,
367 int db_num));
368void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx);
369int ntb_register_event_callback(struct ntb_device *ndev,
370 void (*event_cb_func)(void *handle,
371 enum ntb_hw_event event));
372void ntb_unregister_event_callback(struct ntb_device *ndev);
373int ntb_get_max_spads(struct ntb_device *ndev);
374int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val);
375int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val);
376int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val);
377int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val);
378resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw);
379void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw);
380u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw);
381void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int idx);
382void *ntb_find_transport(struct pci_dev *pdev);
383
384int ntb_transport_init(struct pci_dev *pdev);
385void ntb_transport_free(void *transport);
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index c5f26cda9f97..9faf1c6029af 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -5,6 +5,7 @@
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
13 * BSD LICENSE 14 * BSD LICENSE
14 * 15 *
15 * Copyright(c) 2012 Intel Corporation. All rights reserved. 16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
16 * 18 *
17 * Redistribution and use in source and binary forms, with or without 19 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions 20 * modification, are permitted provided that the following conditions
@@ -40,7 +42,7 @@
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 * 44 *
43 * Intel PCIe NTB Linux driver 45 * PCIe NTB Transport Linux driver
44 * 46 *
45 * Contact Information: 47 * Contact Information:
46 * Jon Mason <jon.mason@intel.com> 48 * Jon Mason <jon.mason@intel.com>
@@ -56,9 +58,22 @@
56#include <linux/pci.h> 58#include <linux/pci.h>
57#include <linux/slab.h> 59#include <linux/slab.h>
58#include <linux/types.h> 60#include <linux/types.h>
59#include "hw/intel/ntb_hw_intel.h" 61#include "linux/ntb.h"
62#include "linux/ntb_transport.h"
60 63
61#define NTB_TRANSPORT_VERSION 3 64#define NTB_TRANSPORT_VERSION 4
65#define NTB_TRANSPORT_VER "4"
66#define NTB_TRANSPORT_NAME "ntb_transport"
67#define NTB_TRANSPORT_DESC "Software Queue-Pair Transport over NTB"
68
69MODULE_DESCRIPTION(NTB_TRANSPORT_DESC);
70MODULE_VERSION(NTB_TRANSPORT_VER);
71MODULE_LICENSE("Dual BSD/GPL");
72MODULE_AUTHOR("Intel Corporation");
73
74static unsigned long max_mw_size;
75module_param(max_mw_size, ulong, 0644);
76MODULE_PARM_DESC(max_mw_size, "Limit size of large memory windows");
62 77
63static unsigned int transport_mtu = 0x401E; 78static unsigned int transport_mtu = 0x401E;
64module_param(transport_mtu, uint, 0644); 79module_param(transport_mtu, uint, 0644);
@@ -72,10 +87,12 @@ static unsigned int copy_bytes = 1024;
72module_param(copy_bytes, uint, 0644); 87module_param(copy_bytes, uint, 0644);
73MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA"); 88MODULE_PARM_DESC(copy_bytes, "Threshold under which NTB will use the CPU to copy instead of DMA");
74 89
90static struct dentry *nt_debugfs_dir;
91
75struct ntb_queue_entry { 92struct ntb_queue_entry {
76 /* ntb_queue list reference */ 93 /* ntb_queue list reference */
77 struct list_head entry; 94 struct list_head entry;
78 /* pointers to data to be transfered */ 95 /* pointers to data to be transferred */
79 void *cb_data; 96 void *cb_data;
80 void *buf; 97 void *buf;
81 unsigned int len; 98 unsigned int len;
@@ -94,14 +111,16 @@ struct ntb_rx_info {
94}; 111};
95 112
96struct ntb_transport_qp { 113struct ntb_transport_qp {
97 struct ntb_transport *transport; 114 struct ntb_transport_ctx *transport;
98 struct ntb_device *ndev; 115 struct ntb_dev *ndev;
99 void *cb_data; 116 void *cb_data;
100 struct dma_chan *dma_chan; 117 struct dma_chan *dma_chan;
101 118
102 bool client_ready; 119 bool client_ready;
103 bool qp_link; 120 bool link_is_up;
121
104 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */ 122 u8 qp_num; /* Only 64 QP's are allowed. 0-63 */
123 u64 qp_bit;
105 124
106 struct ntb_rx_info __iomem *rx_info; 125 struct ntb_rx_info __iomem *rx_info;
107 struct ntb_rx_info *remote_rx_info; 126 struct ntb_rx_info *remote_rx_info;
@@ -127,6 +146,7 @@ struct ntb_transport_qp {
127 unsigned int rx_max_entry; 146 unsigned int rx_max_entry;
128 unsigned int rx_max_frame; 147 unsigned int rx_max_frame;
129 dma_cookie_t last_cookie; 148 dma_cookie_t last_cookie;
149 struct tasklet_struct rxc_db_work;
130 150
131 void (*event_handler)(void *data, int status); 151 void (*event_handler)(void *data, int status);
132 struct delayed_work link_work; 152 struct delayed_work link_work;
@@ -153,33 +173,44 @@ struct ntb_transport_qp {
153}; 173};
154 174
155struct ntb_transport_mw { 175struct ntb_transport_mw {
156 size_t size; 176 phys_addr_t phys_addr;
177 resource_size_t phys_size;
178 resource_size_t xlat_align;
179 resource_size_t xlat_align_size;
180 void __iomem *vbase;
181 size_t xlat_size;
182 size_t buff_size;
157 void *virt_addr; 183 void *virt_addr;
158 dma_addr_t dma_addr; 184 dma_addr_t dma_addr;
159}; 185};
160 186
161struct ntb_transport_client_dev { 187struct ntb_transport_client_dev {
162 struct list_head entry; 188 struct list_head entry;
189 struct ntb_transport_ctx *nt;
163 struct device dev; 190 struct device dev;
164}; 191};
165 192
166struct ntb_transport { 193struct ntb_transport_ctx {
167 struct list_head entry; 194 struct list_head entry;
168 struct list_head client_devs; 195 struct list_head client_devs;
169 196
170 struct ntb_device *ndev; 197 struct ntb_dev *ndev;
171 struct ntb_transport_mw *mw; 198
172 struct ntb_transport_qp *qps; 199 struct ntb_transport_mw *mw_vec;
173 unsigned int max_qps; 200 struct ntb_transport_qp *qp_vec;
174 unsigned long qp_bitmap; 201 unsigned int mw_count;
175 bool transport_link; 202 unsigned int qp_count;
203 u64 qp_bitmap;
204 u64 qp_bitmap_free;
205
206 bool link_is_up;
176 struct delayed_work link_work; 207 struct delayed_work link_work;
177 struct work_struct link_cleanup; 208 struct work_struct link_cleanup;
178}; 209};
179 210
180enum { 211enum {
181 DESC_DONE_FLAG = 1 << 0, 212 DESC_DONE_FLAG = BIT(0),
182 LINK_DOWN_FLAG = 1 << 1, 213 LINK_DOWN_FLAG = BIT(1),
183}; 214};
184 215
185struct ntb_payload_header { 216struct ntb_payload_header {
@@ -200,68 +231,69 @@ enum {
200 MAX_SPAD, 231 MAX_SPAD,
201}; 232};
202 233
203#define QP_TO_MW(ndev, qp) ((qp) % ntb_max_mw(ndev)) 234#define dev_client_dev(__dev) \
235 container_of((__dev), struct ntb_transport_client_dev, dev)
236
237#define drv_client(__drv) \
238 container_of((__drv), struct ntb_transport_client, driver)
239
240#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
204#define NTB_QP_DEF_NUM_ENTRIES 100 241#define NTB_QP_DEF_NUM_ENTRIES 100
205#define NTB_LINK_DOWN_TIMEOUT 10 242#define NTB_LINK_DOWN_TIMEOUT 10
206 243
207static int ntb_match_bus(struct device *dev, struct device_driver *drv) 244static void ntb_transport_rxc_db(unsigned long data);
245static const struct ntb_ctx_ops ntb_transport_ops;
246static struct ntb_client ntb_transport_client;
247
248static int ntb_transport_bus_match(struct device *dev,
249 struct device_driver *drv)
208{ 250{
209 return !strncmp(dev_name(dev), drv->name, strlen(drv->name)); 251 return !strncmp(dev_name(dev), drv->name, strlen(drv->name));
210} 252}
211 253
212static int ntb_client_probe(struct device *dev) 254static int ntb_transport_bus_probe(struct device *dev)
213{ 255{
214 const struct ntb_client *drv = container_of(dev->driver, 256 const struct ntb_transport_client *client;
215 struct ntb_client, driver);
216 struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev);
217 int rc = -EINVAL; 257 int rc = -EINVAL;
218 258
219 get_device(dev); 259 get_device(dev);
220 if (drv && drv->probe) 260
221 rc = drv->probe(pdev); 261 client = drv_client(dev->driver);
262 rc = client->probe(dev);
222 if (rc) 263 if (rc)
223 put_device(dev); 264 put_device(dev);
224 265
225 return rc; 266 return rc;
226} 267}
227 268
228static int ntb_client_remove(struct device *dev) 269static int ntb_transport_bus_remove(struct device *dev)
229{ 270{
230 const struct ntb_client *drv = container_of(dev->driver, 271 const struct ntb_transport_client *client;
231 struct ntb_client, driver);
232 struct pci_dev *pdev = container_of(dev->parent, struct pci_dev, dev);
233 272
234 if (drv && drv->remove) 273 client = drv_client(dev->driver);
235 drv->remove(pdev); 274 client->remove(dev);
236 275
237 put_device(dev); 276 put_device(dev);
238 277
239 return 0; 278 return 0;
240} 279}
241 280
242static struct bus_type ntb_bus_type = { 281static struct bus_type ntb_transport_bus = {
243 .name = "ntb_bus", 282 .name = "ntb_transport",
244 .match = ntb_match_bus, 283 .match = ntb_transport_bus_match,
245 .probe = ntb_client_probe, 284 .probe = ntb_transport_bus_probe,
246 .remove = ntb_client_remove, 285 .remove = ntb_transport_bus_remove,
247}; 286};
248 287
249static LIST_HEAD(ntb_transport_list); 288static LIST_HEAD(ntb_transport_list);
250 289
251static int ntb_bus_init(struct ntb_transport *nt) 290static int ntb_bus_init(struct ntb_transport_ctx *nt)
252{ 291{
253 if (list_empty(&ntb_transport_list)) {
254 int rc = bus_register(&ntb_bus_type);
255 if (rc)
256 return rc;
257 }
258
259 list_add(&nt->entry, &ntb_transport_list); 292 list_add(&nt->entry, &ntb_transport_list);
260
261 return 0; 293 return 0;
262} 294}
263 295
264static void ntb_bus_remove(struct ntb_transport *nt) 296static void ntb_bus_remove(struct ntb_transport_ctx *nt)
265{ 297{
266 struct ntb_transport_client_dev *client_dev, *cd; 298 struct ntb_transport_client_dev *client_dev, *cd;
267 299
@@ -273,29 +305,26 @@ static void ntb_bus_remove(struct ntb_transport *nt)
273 } 305 }
274 306
275 list_del(&nt->entry); 307 list_del(&nt->entry);
276
277 if (list_empty(&ntb_transport_list))
278 bus_unregister(&ntb_bus_type);
279} 308}
280 309
281static void ntb_client_release(struct device *dev) 310static void ntb_transport_client_release(struct device *dev)
282{ 311{
283 struct ntb_transport_client_dev *client_dev; 312 struct ntb_transport_client_dev *client_dev;
284 client_dev = container_of(dev, struct ntb_transport_client_dev, dev);
285 313
314 client_dev = dev_client_dev(dev);
286 kfree(client_dev); 315 kfree(client_dev);
287} 316}
288 317
289/** 318/**
290 * ntb_unregister_client_dev - Unregister NTB client device 319 * ntb_transport_unregister_client_dev - Unregister NTB client device
291 * @device_name: Name of NTB client device 320 * @device_name: Name of NTB client device
292 * 321 *
293 * Unregister an NTB client device with the NTB transport layer 322 * Unregister an NTB client device with the NTB transport layer
294 */ 323 */
295void ntb_unregister_client_dev(char *device_name) 324void ntb_transport_unregister_client_dev(char *device_name)
296{ 325{
297 struct ntb_transport_client_dev *client, *cd; 326 struct ntb_transport_client_dev *client, *cd;
298 struct ntb_transport *nt; 327 struct ntb_transport_ctx *nt;
299 328
300 list_for_each_entry(nt, &ntb_transport_list, entry) 329 list_for_each_entry(nt, &ntb_transport_list, entry)
301 list_for_each_entry_safe(client, cd, &nt->client_devs, entry) 330 list_for_each_entry_safe(client, cd, &nt->client_devs, entry)
@@ -305,18 +334,18 @@ void ntb_unregister_client_dev(char *device_name)
305 device_unregister(&client->dev); 334 device_unregister(&client->dev);
306 } 335 }
307} 336}
308EXPORT_SYMBOL_GPL(ntb_unregister_client_dev); 337EXPORT_SYMBOL_GPL(ntb_transport_unregister_client_dev);
309 338
310/** 339/**
311 * ntb_register_client_dev - Register NTB client device 340 * ntb_transport_register_client_dev - Register NTB client device
312 * @device_name: Name of NTB client device 341 * @device_name: Name of NTB client device
313 * 342 *
314 * Register an NTB client device with the NTB transport layer 343 * Register an NTB client device with the NTB transport layer
315 */ 344 */
316int ntb_register_client_dev(char *device_name) 345int ntb_transport_register_client_dev(char *device_name)
317{ 346{
318 struct ntb_transport_client_dev *client_dev; 347 struct ntb_transport_client_dev *client_dev;
319 struct ntb_transport *nt; 348 struct ntb_transport_ctx *nt;
320 int rc, i = 0; 349 int rc, i = 0;
321 350
322 if (list_empty(&ntb_transport_list)) 351 if (list_empty(&ntb_transport_list))
@@ -325,7 +354,7 @@ int ntb_register_client_dev(char *device_name)
325 list_for_each_entry(nt, &ntb_transport_list, entry) { 354 list_for_each_entry(nt, &ntb_transport_list, entry) {
326 struct device *dev; 355 struct device *dev;
327 356
328 client_dev = kzalloc(sizeof(struct ntb_transport_client_dev), 357 client_dev = kzalloc(sizeof(*client_dev),
329 GFP_KERNEL); 358 GFP_KERNEL);
330 if (!client_dev) { 359 if (!client_dev) {
331 rc = -ENOMEM; 360 rc = -ENOMEM;
@@ -336,9 +365,9 @@ int ntb_register_client_dev(char *device_name)
336 365
337 /* setup and register client devices */ 366 /* setup and register client devices */
338 dev_set_name(dev, "%s%d", device_name, i); 367 dev_set_name(dev, "%s%d", device_name, i);
339 dev->bus = &ntb_bus_type; 368 dev->bus = &ntb_transport_bus;
340 dev->release = ntb_client_release; 369 dev->release = ntb_transport_client_release;
341 dev->parent = &ntb_query_pdev(nt->ndev)->dev; 370 dev->parent = &nt->ndev->dev;
342 371
343 rc = device_register(dev); 372 rc = device_register(dev);
344 if (rc) { 373 if (rc) {
@@ -353,11 +382,11 @@ int ntb_register_client_dev(char *device_name)
353 return 0; 382 return 0;
354 383
355err: 384err:
356 ntb_unregister_client_dev(device_name); 385 ntb_transport_unregister_client_dev(device_name);
357 386
358 return rc; 387 return rc;
359} 388}
360EXPORT_SYMBOL_GPL(ntb_register_client_dev); 389EXPORT_SYMBOL_GPL(ntb_transport_register_client_dev);
361 390
362/** 391/**
363 * ntb_transport_register_client - Register NTB client driver 392 * ntb_transport_register_client - Register NTB client driver
@@ -367,9 +396,9 @@ EXPORT_SYMBOL_GPL(ntb_register_client_dev);
367 * 396 *
368 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 397 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
369 */ 398 */
370int ntb_transport_register_client(struct ntb_client *drv) 399int ntb_transport_register_client(struct ntb_transport_client *drv)
371{ 400{
372 drv->driver.bus = &ntb_bus_type; 401 drv->driver.bus = &ntb_transport_bus;
373 402
374 if (list_empty(&ntb_transport_list)) 403 if (list_empty(&ntb_transport_list))
375 return -ENODEV; 404 return -ENODEV;
@@ -386,7 +415,7 @@ EXPORT_SYMBOL_GPL(ntb_transport_register_client);
386 * 415 *
387 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 416 * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
388 */ 417 */
389void ntb_transport_unregister_client(struct ntb_client *drv) 418void ntb_transport_unregister_client(struct ntb_transport_client *drv)
390{ 419{
391 driver_unregister(&drv->driver); 420 driver_unregister(&drv->driver);
392} 421}
@@ -452,8 +481,8 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
452 "tx_max_entry - \t%u\n", qp->tx_max_entry); 481 "tx_max_entry - \t%u\n", qp->tx_max_entry);
453 482
454 out_offset += snprintf(buf + out_offset, out_count - out_offset, 483 out_offset += snprintf(buf + out_offset, out_count - out_offset,
455 "\nQP Link %s\n", (qp->qp_link == NTB_LINK_UP) ? 484 "\nQP Link %s\n",
456 "Up" : "Down"); 485 qp->link_is_up ? "Up" : "Down");
457 if (out_offset > out_count) 486 if (out_offset > out_count)
458 out_offset = out_count; 487 out_offset = out_count;
459 488
@@ -497,26 +526,31 @@ out:
497 return entry; 526 return entry;
498} 527}
499 528
500static void ntb_transport_setup_qp_mw(struct ntb_transport *nt, 529static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
501 unsigned int qp_num) 530 unsigned int qp_num)
502{ 531{
503 struct ntb_transport_qp *qp = &nt->qps[qp_num]; 532 struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
533 struct ntb_transport_mw *mw;
504 unsigned int rx_size, num_qps_mw; 534 unsigned int rx_size, num_qps_mw;
505 u8 mw_num, mw_max; 535 unsigned int mw_num, mw_count, qp_count;
506 unsigned int i; 536 unsigned int i;
507 537
508 mw_max = ntb_max_mw(nt->ndev); 538 mw_count = nt->mw_count;
509 mw_num = QP_TO_MW(nt->ndev, qp_num); 539 qp_count = nt->qp_count;
510 540
511 WARN_ON(nt->mw[mw_num].virt_addr == NULL); 541 mw_num = QP_TO_MW(nt, qp_num);
542 mw = &nt->mw_vec[mw_num];
543
544 if (!mw->virt_addr)
545 return -ENOMEM;
512 546
513 if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max) 547 if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
514 num_qps_mw = nt->max_qps / mw_max + 1; 548 num_qps_mw = qp_count / mw_count + 1;
515 else 549 else
516 num_qps_mw = nt->max_qps / mw_max; 550 num_qps_mw = qp_count / mw_count;
517 551
518 rx_size = (unsigned int) nt->mw[mw_num].size / num_qps_mw; 552 rx_size = (unsigned int)mw->xlat_size / num_qps_mw;
519 qp->rx_buff = nt->mw[mw_num].virt_addr + qp_num / mw_max * rx_size; 553 qp->rx_buff = mw->virt_addr + rx_size * qp_num / mw_count;
520 rx_size -= sizeof(struct ntb_rx_info); 554 rx_size -= sizeof(struct ntb_rx_info);
521 555
522 qp->remote_rx_info = qp->rx_buff + rx_size; 556 qp->remote_rx_info = qp->rx_buff + rx_size;
@@ -530,49 +564,63 @@ static void ntb_transport_setup_qp_mw(struct ntb_transport *nt,
530 564
531 /* setup the hdr offsets with 0's */ 565 /* setup the hdr offsets with 0's */
532 for (i = 0; i < qp->rx_max_entry; i++) { 566 for (i = 0; i < qp->rx_max_entry; i++) {
533 void *offset = qp->rx_buff + qp->rx_max_frame * (i + 1) - 567 void *offset = (qp->rx_buff + qp->rx_max_frame * (i + 1) -
534 sizeof(struct ntb_payload_header); 568 sizeof(struct ntb_payload_header));
535 memset(offset, 0, sizeof(struct ntb_payload_header)); 569 memset(offset, 0, sizeof(struct ntb_payload_header));
536 } 570 }
537 571
538 qp->rx_pkts = 0; 572 qp->rx_pkts = 0;
539 qp->tx_pkts = 0; 573 qp->tx_pkts = 0;
540 qp->tx_index = 0; 574 qp->tx_index = 0;
575
576 return 0;
541} 577}
542 578
543static void ntb_free_mw(struct ntb_transport *nt, int num_mw) 579static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
544{ 580{
545 struct ntb_transport_mw *mw = &nt->mw[num_mw]; 581 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
546 struct pci_dev *pdev = ntb_query_pdev(nt->ndev); 582 struct pci_dev *pdev = nt->ndev->pdev;
547 583
548 if (!mw->virt_addr) 584 if (!mw->virt_addr)
549 return; 585 return;
550 586
551 dma_free_coherent(&pdev->dev, mw->size, mw->virt_addr, mw->dma_addr); 587 ntb_mw_clear_trans(nt->ndev, num_mw);
588 dma_free_coherent(&pdev->dev, mw->buff_size,
589 mw->virt_addr, mw->dma_addr);
590 mw->xlat_size = 0;
591 mw->buff_size = 0;
552 mw->virt_addr = NULL; 592 mw->virt_addr = NULL;
553} 593}
554 594
555static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size) 595static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,
596 unsigned int size)
556{ 597{
557 struct ntb_transport_mw *mw = &nt->mw[num_mw]; 598 struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
558 struct pci_dev *pdev = ntb_query_pdev(nt->ndev); 599 struct pci_dev *pdev = nt->ndev->pdev;
600 unsigned int xlat_size, buff_size;
601 int rc;
602
603 xlat_size = round_up(size, mw->xlat_align_size);
604 buff_size = round_up(size, mw->xlat_align);
559 605
560 /* No need to re-setup */ 606 /* No need to re-setup */
561 if (mw->size == ALIGN(size, 4096)) 607 if (mw->xlat_size == xlat_size)
562 return 0; 608 return 0;
563 609
564 if (mw->size != 0) 610 if (mw->buff_size)
565 ntb_free_mw(nt, num_mw); 611 ntb_free_mw(nt, num_mw);
566 612
567 /* Alloc memory for receiving data. Must be 4k aligned */ 613 /* Alloc memory for receiving data. Must be aligned */
568 mw->size = ALIGN(size, 4096); 614 mw->xlat_size = xlat_size;
615 mw->buff_size = buff_size;
569 616
570 mw->virt_addr = dma_alloc_coherent(&pdev->dev, mw->size, &mw->dma_addr, 617 mw->virt_addr = dma_alloc_coherent(&pdev->dev, buff_size,
571 GFP_KERNEL); 618 &mw->dma_addr, GFP_KERNEL);
572 if (!mw->virt_addr) { 619 if (!mw->virt_addr) {
573 mw->size = 0; 620 mw->xlat_size = 0;
574 dev_err(&pdev->dev, "Unable to allocate MW buffer of size %d\n", 621 mw->buff_size = 0;
575 (int) mw->size); 622 dev_err(&pdev->dev, "Unable to alloc MW buff of size %d\n",
623 buff_size);
576 return -ENOMEM; 624 return -ENOMEM;
577 } 625 }
578 626
@@ -582,34 +630,39 @@ static int ntb_set_mw(struct ntb_transport *nt, int num_mw, unsigned int size)
582 * is a requirement of the hardware. It is recommended to setup CMA 630 * is a requirement of the hardware. It is recommended to setup CMA
583 * for BAR sizes equal or greater than 4MB. 631 * for BAR sizes equal or greater than 4MB.
584 */ 632 */
585 if (!IS_ALIGNED(mw->dma_addr, mw->size)) { 633 if (!IS_ALIGNED(mw->dma_addr, mw->xlat_align)) {
586 dev_err(&pdev->dev, "DMA memory %pad not aligned to BAR size\n", 634 dev_err(&pdev->dev, "DMA memory %pad is not aligned\n",
587 &mw->dma_addr); 635 &mw->dma_addr);
588 ntb_free_mw(nt, num_mw); 636 ntb_free_mw(nt, num_mw);
589 return -ENOMEM; 637 return -ENOMEM;
590 } 638 }
591 639
592 /* Notify HW the memory location of the receive buffer */ 640 /* Notify HW the memory location of the receive buffer */
593 ntb_set_mw_addr(nt->ndev, num_mw, mw->dma_addr); 641 rc = ntb_mw_set_trans(nt->ndev, num_mw, mw->dma_addr, mw->xlat_size);
642 if (rc) {
643 dev_err(&pdev->dev, "Unable to set mw%d translation", num_mw);
644 ntb_free_mw(nt, num_mw);
645 return -EIO;
646 }
594 647
595 return 0; 648 return 0;
596} 649}
597 650
598static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 651static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
599{ 652{
600 struct ntb_transport *nt = qp->transport; 653 struct ntb_transport_ctx *nt = qp->transport;
601 struct pci_dev *pdev = ntb_query_pdev(nt->ndev); 654 struct pci_dev *pdev = nt->ndev->pdev;
602 655
603 if (qp->qp_link == NTB_LINK_DOWN) { 656 if (qp->link_is_up) {
604 cancel_delayed_work_sync(&qp->link_work); 657 cancel_delayed_work_sync(&qp->link_work);
605 return; 658 return;
606 } 659 }
607 660
608 if (qp->event_handler)
609 qp->event_handler(qp->cb_data, NTB_LINK_DOWN);
610
611 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); 661 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);
612 qp->qp_link = NTB_LINK_DOWN; 662 qp->link_is_up = false;
663
664 if (qp->event_handler)
665 qp->event_handler(qp->cb_data, qp->link_is_up);
613} 666}
614 667
615static void ntb_qp_link_cleanup_work(struct work_struct *work) 668static void ntb_qp_link_cleanup_work(struct work_struct *work)
@@ -617,11 +670,11 @@ static void ntb_qp_link_cleanup_work(struct work_struct *work)
617 struct ntb_transport_qp *qp = container_of(work, 670 struct ntb_transport_qp *qp = container_of(work,
618 struct ntb_transport_qp, 671 struct ntb_transport_qp,
619 link_cleanup); 672 link_cleanup);
620 struct ntb_transport *nt = qp->transport; 673 struct ntb_transport_ctx *nt = qp->transport;
621 674
622 ntb_qp_link_cleanup(qp); 675 ntb_qp_link_cleanup(qp);
623 676
624 if (nt->transport_link == NTB_LINK_UP) 677 if (nt->link_is_up)
625 schedule_delayed_work(&qp->link_work, 678 schedule_delayed_work(&qp->link_work,
626 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 679 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
627} 680}
@@ -631,180 +684,132 @@ static void ntb_qp_link_down(struct ntb_transport_qp *qp)
631 schedule_work(&qp->link_cleanup); 684 schedule_work(&qp->link_cleanup);
632} 685}
633 686
634static void ntb_transport_link_cleanup(struct ntb_transport *nt) 687static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)
635{ 688{
689 struct ntb_transport_qp *qp;
690 u64 qp_bitmap_alloc;
636 int i; 691 int i;
637 692
693 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;
694
638 /* Pass along the info to any clients */ 695 /* Pass along the info to any clients */
639 for (i = 0; i < nt->max_qps; i++) 696 for (i = 0; i < nt->qp_count; i++)
640 if (!test_bit(i, &nt->qp_bitmap)) 697 if (qp_bitmap_alloc & BIT_ULL(i)) {
641 ntb_qp_link_cleanup(&nt->qps[i]); 698 qp = &nt->qp_vec[i];
699 ntb_qp_link_cleanup(qp);
700 cancel_work_sync(&qp->link_cleanup);
701 cancel_delayed_work_sync(&qp->link_work);
702 }
642 703
643 if (nt->transport_link == NTB_LINK_DOWN) 704 if (!nt->link_is_up)
644 cancel_delayed_work_sync(&nt->link_work); 705 cancel_delayed_work_sync(&nt->link_work);
645 else
646 nt->transport_link = NTB_LINK_DOWN;
647 706
648 /* The scratchpad registers keep the values if the remote side 707 /* The scratchpad registers keep the values if the remote side
649 * goes down, blast them now to give them a sane value the next 708 * goes down, blast them now to give them a sane value the next
650 * time they are accessed 709 * time they are accessed
651 */ 710 */
652 for (i = 0; i < MAX_SPAD; i++) 711 for (i = 0; i < MAX_SPAD; i++)
653 ntb_write_local_spad(nt->ndev, i, 0); 712 ntb_spad_write(nt->ndev, i, 0);
654} 713}
655 714
656static void ntb_transport_link_cleanup_work(struct work_struct *work) 715static void ntb_transport_link_cleanup_work(struct work_struct *work)
657{ 716{
658 struct ntb_transport *nt = container_of(work, struct ntb_transport, 717 struct ntb_transport_ctx *nt =
659 link_cleanup); 718 container_of(work, struct ntb_transport_ctx, link_cleanup);
660 719
661 ntb_transport_link_cleanup(nt); 720 ntb_transport_link_cleanup(nt);
662} 721}
663 722
664static void ntb_transport_event_callback(void *data, enum ntb_hw_event event) 723static void ntb_transport_event_callback(void *data)
665{ 724{
666 struct ntb_transport *nt = data; 725 struct ntb_transport_ctx *nt = data;
667 726
668 switch (event) { 727 if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)
669 case NTB_EVENT_HW_LINK_UP:
670 schedule_delayed_work(&nt->link_work, 0); 728 schedule_delayed_work(&nt->link_work, 0);
671 break; 729 else
672 case NTB_EVENT_HW_LINK_DOWN:
673 schedule_work(&nt->link_cleanup); 730 schedule_work(&nt->link_cleanup);
674 break;
675 default:
676 BUG();
677 }
678} 731}
679 732
680static void ntb_transport_link_work(struct work_struct *work) 733static void ntb_transport_link_work(struct work_struct *work)
681{ 734{
682 struct ntb_transport *nt = container_of(work, struct ntb_transport, 735 struct ntb_transport_ctx *nt =
683 link_work.work); 736 container_of(work, struct ntb_transport_ctx, link_work.work);
684 struct ntb_device *ndev = nt->ndev; 737 struct ntb_dev *ndev = nt->ndev;
685 struct pci_dev *pdev = ntb_query_pdev(ndev); 738 struct pci_dev *pdev = ndev->pdev;
739 resource_size_t size;
686 u32 val; 740 u32 val;
687 int rc, i; 741 int rc, i, spad;
688 742
689 /* send the local info, in the opposite order of the way we read it */ 743 /* send the local info, in the opposite order of the way we read it */
690 for (i = 0; i < ntb_max_mw(ndev); i++) { 744 for (i = 0; i < nt->mw_count; i++) {
691 rc = ntb_write_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), 745 size = nt->mw_vec[i].phys_size;
692 ntb_get_mw_size(ndev, i) >> 32);
693 if (rc) {
694 dev_err(&pdev->dev, "Error writing %u to remote spad %d\n",
695 (u32)(ntb_get_mw_size(ndev, i) >> 32),
696 MW0_SZ_HIGH + (i * 2));
697 goto out;
698 }
699 746
700 rc = ntb_write_remote_spad(ndev, MW0_SZ_LOW + (i * 2), 747 if (max_mw_size && size > max_mw_size)
701 (u32) ntb_get_mw_size(ndev, i)); 748 size = max_mw_size;
702 if (rc) {
703 dev_err(&pdev->dev, "Error writing %u to remote spad %d\n",
704 (u32) ntb_get_mw_size(ndev, i),
705 MW0_SZ_LOW + (i * 2));
706 goto out;
707 }
708 }
709 749
710 rc = ntb_write_remote_spad(ndev, NUM_MWS, ntb_max_mw(ndev)); 750 spad = MW0_SZ_HIGH + (i * 2);
711 if (rc) { 751 ntb_peer_spad_write(ndev, spad, (u32)(size >> 32));
712 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
713 ntb_max_mw(ndev), NUM_MWS);
714 goto out;
715 }
716 752
717 rc = ntb_write_remote_spad(ndev, NUM_QPS, nt->max_qps); 753 spad = MW0_SZ_LOW + (i * 2);
718 if (rc) { 754 ntb_peer_spad_write(ndev, spad, (u32)size);
719 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
720 nt->max_qps, NUM_QPS);
721 goto out;
722 } 755 }
723 756
724 rc = ntb_write_remote_spad(ndev, VERSION, NTB_TRANSPORT_VERSION); 757 ntb_peer_spad_write(ndev, NUM_MWS, nt->mw_count);
725 if (rc) {
726 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
727 NTB_TRANSPORT_VERSION, VERSION);
728 goto out;
729 }
730 758
731 /* Query the remote side for its info */ 759 ntb_peer_spad_write(ndev, NUM_QPS, nt->qp_count);
732 rc = ntb_read_remote_spad(ndev, VERSION, &val);
733 if (rc) {
734 dev_err(&pdev->dev, "Error reading remote spad %d\n", VERSION);
735 goto out;
736 }
737 760
738 if (val != NTB_TRANSPORT_VERSION) 761 ntb_peer_spad_write(ndev, VERSION, NTB_TRANSPORT_VERSION);
739 goto out;
740 dev_dbg(&pdev->dev, "Remote version = %d\n", val);
741 762
742 rc = ntb_read_remote_spad(ndev, NUM_QPS, &val); 763 /* Query the remote side for its info */
743 if (rc) { 764 val = ntb_peer_spad_read(ndev, VERSION);
744 dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_QPS); 765 dev_dbg(&pdev->dev, "Remote version = %d\n", val);
766 if (val != NTB_TRANSPORT_VERSION)
745 goto out; 767 goto out;
746 }
747 768
748 if (val != nt->max_qps) 769 val = ntb_peer_spad_read(ndev, NUM_QPS);
749 goto out;
750 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val); 770 dev_dbg(&pdev->dev, "Remote max number of qps = %d\n", val);
751 771 if (val != nt->qp_count)
752 rc = ntb_read_remote_spad(ndev, NUM_MWS, &val);
753 if (rc) {
754 dev_err(&pdev->dev, "Error reading remote spad %d\n", NUM_MWS);
755 goto out; 772 goto out;
756 }
757 773
758 if (val != ntb_max_mw(ndev)) 774 val = ntb_peer_spad_read(ndev, NUM_MWS);
759 goto out;
760 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val); 775 dev_dbg(&pdev->dev, "Remote number of mws = %d\n", val);
776 if (val != nt->mw_count)
777 goto out;
761 778
762 for (i = 0; i < ntb_max_mw(ndev); i++) { 779 for (i = 0; i < nt->mw_count; i++) {
763 u64 val64; 780 u64 val64;
764 781
765 rc = ntb_read_remote_spad(ndev, MW0_SZ_HIGH + (i * 2), &val); 782 val = ntb_peer_spad_read(ndev, MW0_SZ_HIGH + (i * 2));
766 if (rc) { 783 val64 = (u64)val << 32;
767 dev_err(&pdev->dev, "Error reading remote spad %d\n",
768 MW0_SZ_HIGH + (i * 2));
769 goto out1;
770 }
771
772 val64 = (u64) val << 32;
773
774 rc = ntb_read_remote_spad(ndev, MW0_SZ_LOW + (i * 2), &val);
775 if (rc) {
776 dev_err(&pdev->dev, "Error reading remote spad %d\n",
777 MW0_SZ_LOW + (i * 2));
778 goto out1;
779 }
780 784
785 val = ntb_peer_spad_read(ndev, MW0_SZ_LOW + (i * 2));
781 val64 |= val; 786 val64 |= val;
782 787
783 dev_dbg(&pdev->dev, "Remote MW%d size = %llu\n", i, val64); 788 dev_dbg(&pdev->dev, "Remote MW%d size = %#llx\n", i, val64);
784 789
785 rc = ntb_set_mw(nt, i, val64); 790 rc = ntb_set_mw(nt, i, val64);
786 if (rc) 791 if (rc)
787 goto out1; 792 goto out1;
788 } 793 }
789 794
790 nt->transport_link = NTB_LINK_UP; 795 nt->link_is_up = true;
791 796
792 for (i = 0; i < nt->max_qps; i++) { 797 for (i = 0; i < nt->qp_count; i++) {
793 struct ntb_transport_qp *qp = &nt->qps[i]; 798 struct ntb_transport_qp *qp = &nt->qp_vec[i];
794 799
795 ntb_transport_setup_qp_mw(nt, i); 800 ntb_transport_setup_qp_mw(nt, i);
796 801
797 if (qp->client_ready == NTB_LINK_UP) 802 if (qp->client_ready)
798 schedule_delayed_work(&qp->link_work, 0); 803 schedule_delayed_work(&qp->link_work, 0);
799 } 804 }
800 805
801 return; 806 return;
802 807
803out1: 808out1:
804 for (i = 0; i < ntb_max_mw(ndev); i++) 809 for (i = 0; i < nt->mw_count; i++)
805 ntb_free_mw(nt, i); 810 ntb_free_mw(nt, i);
806out: 811out:
807 if (ntb_hw_link_status(ndev)) 812 if (ntb_link_is_up(ndev, NULL, NULL) == 1)
808 schedule_delayed_work(&nt->link_work, 813 schedule_delayed_work(&nt->link_work,
809 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 814 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
810} 815}
@@ -814,73 +819,73 @@ static void ntb_qp_link_work(struct work_struct *work)
814 struct ntb_transport_qp *qp = container_of(work, 819 struct ntb_transport_qp *qp = container_of(work,
815 struct ntb_transport_qp, 820 struct ntb_transport_qp,
816 link_work.work); 821 link_work.work);
817 struct pci_dev *pdev = ntb_query_pdev(qp->ndev); 822 struct pci_dev *pdev = qp->ndev->pdev;
818 struct ntb_transport *nt = qp->transport; 823 struct ntb_transport_ctx *nt = qp->transport;
819 int rc, val; 824 int val;
820 825
821 WARN_ON(nt->transport_link != NTB_LINK_UP); 826 WARN_ON(!nt->link_is_up);
822 827
823 rc = ntb_read_local_spad(nt->ndev, QP_LINKS, &val); 828 val = ntb_spad_read(nt->ndev, QP_LINKS);
824 if (rc) {
825 dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS);
826 return;
827 }
828 829
829 rc = ntb_write_remote_spad(nt->ndev, QP_LINKS, val | 1 << qp->qp_num); 830 ntb_peer_spad_write(nt->ndev, QP_LINKS, val | BIT(qp->qp_num));
830 if (rc)
831 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
832 val | 1 << qp->qp_num, QP_LINKS);
833 831
834 /* query remote spad for qp ready bits */ 832 /* query remote spad for qp ready bits */
835 rc = ntb_read_remote_spad(nt->ndev, QP_LINKS, &val); 833 ntb_peer_spad_read(nt->ndev, QP_LINKS);
836 if (rc)
837 dev_err(&pdev->dev, "Error reading remote spad %d\n", QP_LINKS);
838
839 dev_dbg(&pdev->dev, "Remote QP link status = %x\n", val); 834 dev_dbg(&pdev->dev, "Remote QP link status = %x\n", val);
840 835
841 /* See if the remote side is up */ 836 /* See if the remote side is up */
842 if (1 << qp->qp_num & val) { 837 if (val & BIT(qp->qp_num)) {
843 qp->qp_link = NTB_LINK_UP;
844
845 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num); 838 dev_info(&pdev->dev, "qp %d: Link Up\n", qp->qp_num);
839 qp->link_is_up = true;
840
846 if (qp->event_handler) 841 if (qp->event_handler)
847 qp->event_handler(qp->cb_data, NTB_LINK_UP); 842 qp->event_handler(qp->cb_data, qp->link_is_up);
848 } else if (nt->transport_link == NTB_LINK_UP) 843 } else if (nt->link_is_up)
849 schedule_delayed_work(&qp->link_work, 844 schedule_delayed_work(&qp->link_work,
850 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT)); 845 msecs_to_jiffies(NTB_LINK_DOWN_TIMEOUT));
851} 846}
852 847
853static int ntb_transport_init_queue(struct ntb_transport *nt, 848static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,
854 unsigned int qp_num) 849 unsigned int qp_num)
855{ 850{
856 struct ntb_transport_qp *qp; 851 struct ntb_transport_qp *qp;
852 struct ntb_transport_mw *mw;
853 phys_addr_t mw_base;
854 resource_size_t mw_size;
857 unsigned int num_qps_mw, tx_size; 855 unsigned int num_qps_mw, tx_size;
858 u8 mw_num, mw_max; 856 unsigned int mw_num, mw_count, qp_count;
859 u64 qp_offset; 857 u64 qp_offset;
860 858
861 mw_max = ntb_max_mw(nt->ndev); 859 mw_count = nt->mw_count;
862 mw_num = QP_TO_MW(nt->ndev, qp_num); 860 qp_count = nt->qp_count;
863 861
864 qp = &nt->qps[qp_num]; 862 mw_num = QP_TO_MW(nt, qp_num);
863 mw = &nt->mw_vec[mw_num];
864
865 qp = &nt->qp_vec[qp_num];
865 qp->qp_num = qp_num; 866 qp->qp_num = qp_num;
866 qp->transport = nt; 867 qp->transport = nt;
867 qp->ndev = nt->ndev; 868 qp->ndev = nt->ndev;
868 qp->qp_link = NTB_LINK_DOWN; 869 qp->link_is_up = false;
869 qp->client_ready = NTB_LINK_DOWN; 870 qp->client_ready = false;
870 qp->event_handler = NULL; 871 qp->event_handler = NULL;
871 872
872 if (nt->max_qps % mw_max && mw_num + 1 < nt->max_qps / mw_max) 873 if (qp_count % mw_count && mw_num + 1 < qp_count / mw_count)
873 num_qps_mw = nt->max_qps / mw_max + 1; 874 num_qps_mw = qp_count / mw_count + 1;
874 else 875 else
875 num_qps_mw = nt->max_qps / mw_max; 876 num_qps_mw = qp_count / mw_count;
877
878 mw_base = nt->mw_vec[mw_num].phys_addr;
879 mw_size = nt->mw_vec[mw_num].phys_size;
876 880
877 tx_size = (unsigned int) ntb_get_mw_size(qp->ndev, mw_num) / num_qps_mw; 881 tx_size = (unsigned int)mw_size / num_qps_mw;
878 qp_offset = qp_num / mw_max * tx_size; 882 qp_offset = tx_size * qp_num / mw_count;
879 qp->tx_mw = ntb_get_mw_vbase(nt->ndev, mw_num) + qp_offset; 883
884 qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;
880 if (!qp->tx_mw) 885 if (!qp->tx_mw)
881 return -EINVAL; 886 return -EINVAL;
882 887
883 qp->tx_mw_phys = ntb_get_mw_base(qp->ndev, mw_num) + qp_offset; 888 qp->tx_mw_phys = mw_base + qp_offset;
884 if (!qp->tx_mw_phys) 889 if (!qp->tx_mw_phys)
885 return -EINVAL; 890 return -EINVAL;
886 891
@@ -891,16 +896,19 @@ static int ntb_transport_init_queue(struct ntb_transport *nt,
891 qp->tx_max_frame = min(transport_mtu, tx_size / 2); 896 qp->tx_max_frame = min(transport_mtu, tx_size / 2);
892 qp->tx_max_entry = tx_size / qp->tx_max_frame; 897 qp->tx_max_entry = tx_size / qp->tx_max_frame;
893 898
894 if (ntb_query_debugfs(nt->ndev)) { 899 if (nt_debugfs_dir) {
895 char debugfs_name[4]; 900 char debugfs_name[4];
896 901
897 snprintf(debugfs_name, 4, "qp%d", qp_num); 902 snprintf(debugfs_name, 4, "qp%d", qp_num);
898 qp->debugfs_dir = debugfs_create_dir(debugfs_name, 903 qp->debugfs_dir = debugfs_create_dir(debugfs_name,
899 ntb_query_debugfs(nt->ndev)); 904 nt_debugfs_dir);
900 905
901 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR, 906 qp->debugfs_stats = debugfs_create_file("stats", S_IRUSR,
902 qp->debugfs_dir, qp, 907 qp->debugfs_dir, qp,
903 &ntb_qp_debugfs_stats); 908 &ntb_qp_debugfs_stats);
909 } else {
910 qp->debugfs_dir = NULL;
911 qp->debugfs_stats = NULL;
904 } 912 }
905 913
906 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work); 914 INIT_DELAYED_WORK(&qp->link_work, ntb_qp_link_work);
@@ -914,46 +922,84 @@ static int ntb_transport_init_queue(struct ntb_transport *nt,
914 INIT_LIST_HEAD(&qp->rx_free_q); 922 INIT_LIST_HEAD(&qp->rx_free_q);
915 INIT_LIST_HEAD(&qp->tx_free_q); 923 INIT_LIST_HEAD(&qp->tx_free_q);
916 924
925 tasklet_init(&qp->rxc_db_work, ntb_transport_rxc_db,
926 (unsigned long)qp);
927
917 return 0; 928 return 0;
918} 929}
919 930
920int ntb_transport_init(struct pci_dev *pdev) 931static int ntb_transport_probe(struct ntb_client *self, struct ntb_dev *ndev)
921{ 932{
922 struct ntb_transport *nt; 933 struct ntb_transport_ctx *nt;
934 struct ntb_transport_mw *mw;
935 unsigned int mw_count, qp_count;
936 u64 qp_bitmap;
923 int rc, i; 937 int rc, i;
924 938
925 nt = kzalloc(sizeof(struct ntb_transport), GFP_KERNEL); 939 if (ntb_db_is_unsafe(ndev))
940 dev_dbg(&ndev->dev,
941 "doorbell is unsafe, proceed anyway...\n");
942 if (ntb_spad_is_unsafe(ndev))
943 dev_dbg(&ndev->dev,
944 "scratchpad is unsafe, proceed anyway...\n");
945
946 nt = kzalloc(sizeof(*nt), GFP_KERNEL);
926 if (!nt) 947 if (!nt)
927 return -ENOMEM; 948 return -ENOMEM;
928 949
929 nt->ndev = ntb_register_transport(pdev, nt); 950 nt->ndev = ndev;
930 if (!nt->ndev) { 951
931 rc = -EIO; 952 mw_count = ntb_mw_count(ndev);
953
954 nt->mw_count = mw_count;
955
956 nt->mw_vec = kcalloc(mw_count, sizeof(*nt->mw_vec), GFP_KERNEL);
957 if (!nt->mw_vec) {
958 rc = -ENOMEM;
932 goto err; 959 goto err;
933 } 960 }
934 961
935 nt->mw = kcalloc(ntb_max_mw(nt->ndev), sizeof(struct ntb_transport_mw), 962 for (i = 0; i < mw_count; i++) {
936 GFP_KERNEL); 963 mw = &nt->mw_vec[i];
937 if (!nt->mw) { 964
938 rc = -ENOMEM; 965 rc = ntb_mw_get_range(ndev, i, &mw->phys_addr, &mw->phys_size,
939 goto err1; 966 &mw->xlat_align, &mw->xlat_align_size);
967 if (rc)
968 goto err1;
969
970 mw->vbase = ioremap(mw->phys_addr, mw->phys_size);
971 if (!mw->vbase) {
972 rc = -ENOMEM;
973 goto err1;
974 }
975
976 mw->buff_size = 0;
977 mw->xlat_size = 0;
978 mw->virt_addr = NULL;
979 mw->dma_addr = 0;
940 } 980 }
941 981
942 if (max_num_clients) 982 qp_bitmap = ntb_db_valid_mask(ndev);
943 nt->max_qps = min(ntb_max_cbs(nt->ndev), max_num_clients); 983
944 else 984 qp_count = ilog2(qp_bitmap);
945 nt->max_qps = min(ntb_max_cbs(nt->ndev), ntb_max_mw(nt->ndev)); 985 if (max_num_clients && max_num_clients < qp_count)
986 qp_count = max_num_clients;
987 else if (mw_count < qp_count)
988 qp_count = mw_count;
989
990 qp_bitmap &= BIT_ULL(qp_count) - 1;
991
992 nt->qp_count = qp_count;
993 nt->qp_bitmap = qp_bitmap;
994 nt->qp_bitmap_free = qp_bitmap;
946 995
947 nt->qps = kcalloc(nt->max_qps, sizeof(struct ntb_transport_qp), 996 nt->qp_vec = kcalloc(qp_count, sizeof(*nt->qp_vec), GFP_KERNEL);
948 GFP_KERNEL); 997 if (!nt->qp_vec) {
949 if (!nt->qps) {
950 rc = -ENOMEM; 998 rc = -ENOMEM;
951 goto err2; 999 goto err2;
952 } 1000 }
953 1001
954 nt->qp_bitmap = ((u64) 1 << nt->max_qps) - 1; 1002 for (i = 0; i < qp_count; i++) {
955
956 for (i = 0; i < nt->max_qps; i++) {
957 rc = ntb_transport_init_queue(nt, i); 1003 rc = ntb_transport_init_queue(nt, i);
958 if (rc) 1004 if (rc)
959 goto err3; 1005 goto err3;
@@ -962,8 +1008,7 @@ int ntb_transport_init(struct pci_dev *pdev)
962 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work); 1008 INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);
963 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work); 1009 INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);
964 1010
965 rc = ntb_register_event_callback(nt->ndev, 1011 rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops);
966 ntb_transport_event_callback);
967 if (rc) 1012 if (rc)
968 goto err3; 1013 goto err3;
969 1014
@@ -972,51 +1017,61 @@ int ntb_transport_init(struct pci_dev *pdev)
972 if (rc) 1017 if (rc)
973 goto err4; 1018 goto err4;
974 1019
975 if (ntb_hw_link_status(nt->ndev)) 1020 nt->link_is_up = false;
976 schedule_delayed_work(&nt->link_work, 0); 1021 ntb_link_enable(ndev, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
1022 ntb_link_event(ndev);
977 1023
978 return 0; 1024 return 0;
979 1025
980err4: 1026err4:
981 ntb_unregister_event_callback(nt->ndev); 1027 ntb_clear_ctx(ndev);
982err3: 1028err3:
983 kfree(nt->qps); 1029 kfree(nt->qp_vec);
984err2: 1030err2:
985 kfree(nt->mw); 1031 kfree(nt->mw_vec);
986err1: 1032err1:
987 ntb_unregister_transport(nt->ndev); 1033 while (i--) {
1034 mw = &nt->mw_vec[i];
1035 iounmap(mw->vbase);
1036 }
988err: 1037err:
989 kfree(nt); 1038 kfree(nt);
990 return rc; 1039 return rc;
991} 1040}
992 1041
993void ntb_transport_free(void *transport) 1042static void ntb_transport_free(struct ntb_client *self, struct ntb_dev *ndev)
994{ 1043{
995 struct ntb_transport *nt = transport; 1044 struct ntb_transport_ctx *nt = ndev->ctx;
996 struct ntb_device *ndev = nt->ndev; 1045 struct ntb_transport_qp *qp;
1046 u64 qp_bitmap_alloc;
997 int i; 1047 int i;
998 1048
999 ntb_transport_link_cleanup(nt); 1049 ntb_transport_link_cleanup(nt);
1050 cancel_work_sync(&nt->link_cleanup);
1051 cancel_delayed_work_sync(&nt->link_work);
1052
1053 qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;
1000 1054
1001 /* verify that all the qp's are freed */ 1055 /* verify that all the qp's are freed */
1002 for (i = 0; i < nt->max_qps; i++) { 1056 for (i = 0; i < nt->qp_count; i++) {
1003 if (!test_bit(i, &nt->qp_bitmap)) 1057 qp = &nt->qp_vec[i];
1004 ntb_transport_free_queue(&nt->qps[i]); 1058 if (qp_bitmap_alloc & BIT_ULL(i))
1005 debugfs_remove_recursive(nt->qps[i].debugfs_dir); 1059 ntb_transport_free_queue(qp);
1060 debugfs_remove_recursive(qp->debugfs_dir);
1006 } 1061 }
1007 1062
1008 ntb_bus_remove(nt); 1063 ntb_link_disable(ndev);
1064 ntb_clear_ctx(ndev);
1009 1065
1010 cancel_delayed_work_sync(&nt->link_work); 1066 ntb_bus_remove(nt);
1011
1012 ntb_unregister_event_callback(ndev);
1013 1067
1014 for (i = 0; i < ntb_max_mw(ndev); i++) 1068 for (i = nt->mw_count; i--; ) {
1015 ntb_free_mw(nt, i); 1069 ntb_free_mw(nt, i);
1070 iounmap(nt->mw_vec[i].vbase);
1071 }
1016 1072
1017 kfree(nt->qps); 1073 kfree(nt->qp_vec);
1018 kfree(nt->mw); 1074 kfree(nt->mw_vec);
1019 ntb_unregister_transport(ndev);
1020 kfree(nt); 1075 kfree(nt);
1021} 1076}
1022 1077
@@ -1028,15 +1083,13 @@ static void ntb_rx_copy_callback(void *data)
1028 unsigned int len = entry->len; 1083 unsigned int len = entry->len;
1029 struct ntb_payload_header *hdr = entry->rx_hdr; 1084 struct ntb_payload_header *hdr = entry->rx_hdr;
1030 1085
1031 /* Ensure that the data is fully copied out before clearing the flag */
1032 wmb();
1033 hdr->flags = 0; 1086 hdr->flags = 0;
1034 1087
1035 iowrite32(entry->index, &qp->rx_info->entry); 1088 iowrite32(entry->index, &qp->rx_info->entry);
1036 1089
1037 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q); 1090 ntb_list_add(&qp->ntb_rx_free_q_lock, &entry->entry, &qp->rx_free_q);
1038 1091
1039 if (qp->rx_handler && qp->client_ready == NTB_LINK_UP) 1092 if (qp->rx_handler && qp->client_ready)
1040 qp->rx_handler(qp, qp->cb_data, cb_data, len); 1093 qp->rx_handler(qp, qp->cb_data, cb_data, len);
1041} 1094}
1042 1095
@@ -1047,6 +1100,9 @@ static void ntb_memcpy_rx(struct ntb_queue_entry *entry, void *offset)
1047 1100
1048 memcpy(buf, offset, len); 1101 memcpy(buf, offset, len);
1049 1102
1103 /* Ensure that the data is fully copied out before clearing the flag */
1104 wmb();
1105
1050 ntb_rx_copy_callback(entry); 1106 ntb_rx_copy_callback(entry);
1051} 1107}
1052 1108
@@ -1071,8 +1127,8 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset,
1071 goto err_wait; 1127 goto err_wait;
1072 1128
1073 device = chan->device; 1129 device = chan->device;
1074 pay_off = (size_t) offset & ~PAGE_MASK; 1130 pay_off = (size_t)offset & ~PAGE_MASK;
1075 buff_off = (size_t) buf & ~PAGE_MASK; 1131 buff_off = (size_t)buf & ~PAGE_MASK;
1076 1132
1077 if (!is_dma_copy_aligned(device, pay_off, buff_off, len)) 1133 if (!is_dma_copy_aligned(device, pay_off, buff_off, len))
1078 goto err_wait; 1134 goto err_wait;
@@ -1138,86 +1194,104 @@ static int ntb_process_rxc(struct ntb_transport_qp *qp)
1138 struct ntb_payload_header *hdr; 1194 struct ntb_payload_header *hdr;
1139 struct ntb_queue_entry *entry; 1195 struct ntb_queue_entry *entry;
1140 void *offset; 1196 void *offset;
1197 int rc;
1141 1198
1142 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index; 1199 offset = qp->rx_buff + qp->rx_max_frame * qp->rx_index;
1143 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header); 1200 hdr = offset + qp->rx_max_frame - sizeof(struct ntb_payload_header);
1144 1201
1145 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 1202 dev_dbg(&qp->ndev->pdev->dev, "qp %d: RX ver %u len %d flags %x\n",
1146 if (!entry) { 1203 qp->qp_num, hdr->ver, hdr->len, hdr->flags);
1147 dev_dbg(&ntb_query_pdev(qp->ndev)->dev,
1148 "no buffer - HDR ver %u, len %d, flags %x\n",
1149 hdr->ver, hdr->len, hdr->flags);
1150 qp->rx_err_no_buf++;
1151 return -ENOMEM;
1152 }
1153 1204
1154 if (!(hdr->flags & DESC_DONE_FLAG)) { 1205 if (!(hdr->flags & DESC_DONE_FLAG)) {
1155 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, 1206 dev_dbg(&qp->ndev->pdev->dev, "done flag not set\n");
1156 &qp->rx_pend_q);
1157 qp->rx_ring_empty++; 1207 qp->rx_ring_empty++;
1158 return -EAGAIN; 1208 return -EAGAIN;
1159 } 1209 }
1160 1210
1161 if (hdr->ver != (u32) qp->rx_pkts) { 1211 if (hdr->flags & LINK_DOWN_FLAG) {
1162 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, 1212 dev_dbg(&qp->ndev->pdev->dev, "link down flag set\n");
1163 "qp %d: version mismatch, expected %llu - got %u\n", 1213 ntb_qp_link_down(qp);
1164 qp->qp_num, qp->rx_pkts, hdr->ver); 1214 hdr->flags = 0;
1165 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, 1215 iowrite32(qp->rx_index, &qp->rx_info->entry);
1166 &qp->rx_pend_q); 1216 return 0;
1217 }
1218
1219 if (hdr->ver != (u32)qp->rx_pkts) {
1220 dev_dbg(&qp->ndev->pdev->dev,
1221 "version mismatch, expected %llu - got %u\n",
1222 qp->rx_pkts, hdr->ver);
1167 qp->rx_err_ver++; 1223 qp->rx_err_ver++;
1168 return -EIO; 1224 return -EIO;
1169 } 1225 }
1170 1226
1171 if (hdr->flags & LINK_DOWN_FLAG) { 1227 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
1172 ntb_qp_link_down(qp); 1228 if (!entry) {
1229 dev_dbg(&qp->ndev->pdev->dev, "no receive buffer\n");
1230 qp->rx_err_no_buf++;
1173 1231
1232 rc = -ENOMEM;
1174 goto err; 1233 goto err;
1175 } 1234 }
1176 1235
1177 dev_dbg(&ntb_query_pdev(qp->ndev)->dev,
1178 "rx offset %u, ver %u - %d payload received, buf size %d\n",
1179 qp->rx_index, hdr->ver, hdr->len, entry->len);
1180
1181 qp->rx_bytes += hdr->len;
1182 qp->rx_pkts++;
1183
1184 if (hdr->len > entry->len) { 1236 if (hdr->len > entry->len) {
1185 qp->rx_err_oflow++; 1237 dev_dbg(&qp->ndev->pdev->dev,
1186 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, 1238 "receive buffer overflow! Wanted %d got %d\n",
1187 "RX overflow! Wanted %d got %d\n",
1188 hdr->len, entry->len); 1239 hdr->len, entry->len);
1240 qp->rx_err_oflow++;
1189 1241
1242 rc = -EIO;
1190 goto err; 1243 goto err;
1191 } 1244 }
1192 1245
1246 dev_dbg(&qp->ndev->pdev->dev,
1247 "RX OK index %u ver %u size %d into buf size %d\n",
1248 qp->rx_index, hdr->ver, hdr->len, entry->len);
1249
1250 qp->rx_bytes += hdr->len;
1251 qp->rx_pkts++;
1252
1193 entry->index = qp->rx_index; 1253 entry->index = qp->rx_index;
1194 entry->rx_hdr = hdr; 1254 entry->rx_hdr = hdr;
1195 1255
1196 ntb_async_rx(entry, offset, hdr->len); 1256 ntb_async_rx(entry, offset, hdr->len);
1197 1257
1198out:
1199 qp->rx_index++; 1258 qp->rx_index++;
1200 qp->rx_index %= qp->rx_max_entry; 1259 qp->rx_index %= qp->rx_max_entry;
1201 1260
1202 return 0; 1261 return 0;
1203 1262
1204err: 1263err:
1205 ntb_list_add(&qp->ntb_rx_pend_q_lock, &entry->entry, &qp->rx_pend_q); 1264 /* FIXME: if this syncrhonous update of the rx_index gets ahead of
1206 /* Ensure that the data is fully copied out before clearing the flag */ 1265 * asyncrhonous ntb_rx_copy_callback of previous entry, there are three
1207 wmb(); 1266 * scenarios:
1267 *
1268 * 1) The peer might miss this update, but observe the update
1269 * from the memcpy completion callback. In this case, the buffer will
1270 * not be freed on the peer to be reused for a different packet. The
1271 * successful rx of a later packet would clear the condition, but the
1272 * condition could persist if several rx fail in a row.
1273 *
1274 * 2) The peer may observe this update before the asyncrhonous copy of
1275 * prior packets is completed. The peer may overwrite the buffers of
1276 * the prior packets before they are copied.
1277 *
1278 * 3) Both: the peer may observe the update, and then observe the index
1279 * decrement by the asynchronous completion callback. Who knows what
1280 * badness that will cause.
1281 */
1208 hdr->flags = 0; 1282 hdr->flags = 0;
1209 iowrite32(qp->rx_index, &qp->rx_info->entry); 1283 iowrite32(qp->rx_index, &qp->rx_info->entry);
1210 1284
1211 goto out; 1285 return rc;
1212} 1286}
1213 1287
1214static int ntb_transport_rxc_db(void *data, int db_num) 1288static void ntb_transport_rxc_db(unsigned long data)
1215{ 1289{
1216 struct ntb_transport_qp *qp = data; 1290 struct ntb_transport_qp *qp = (void *)data;
1217 int rc, i; 1291 int rc, i;
1218 1292
1219 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%s: doorbell %d received\n", 1293 dev_dbg(&qp->ndev->pdev->dev, "%s: doorbell %d received\n",
1220 __func__, db_num); 1294 __func__, qp->qp_num);
1221 1295
1222 /* Limit the number of packets processed in a single interrupt to 1296 /* Limit the number of packets processed in a single interrupt to
1223 * provide fairness to others 1297 * provide fairness to others
@@ -1231,7 +1305,21 @@ static int ntb_transport_rxc_db(void *data, int db_num)
1231 if (qp->dma_chan) 1305 if (qp->dma_chan)
1232 dma_async_issue_pending(qp->dma_chan); 1306 dma_async_issue_pending(qp->dma_chan);
1233 1307
1234 return i; 1308 if (i == qp->rx_max_entry) {
1309 /* there is more work to do */
1310 tasklet_schedule(&qp->rxc_db_work);
1311 } else if (ntb_db_read(qp->ndev) & BIT_ULL(qp->qp_num)) {
1312 /* the doorbell bit is set: clear it */
1313 ntb_db_clear(qp->ndev, BIT_ULL(qp->qp_num));
1314 /* ntb_db_read ensures ntb_db_clear write is committed */
1315 ntb_db_read(qp->ndev);
1316
1317 /* an interrupt may have arrived between finishing
1318 * ntb_process_rxc and clearing the doorbell bit:
1319 * there might be some more work to do.
1320 */
1321 tasklet_schedule(&qp->rxc_db_work);
1322 }
1235} 1323}
1236 1324
1237static void ntb_tx_copy_callback(void *data) 1325static void ntb_tx_copy_callback(void *data)
@@ -1240,11 +1328,9 @@ static void ntb_tx_copy_callback(void *data)
1240 struct ntb_transport_qp *qp = entry->qp; 1328 struct ntb_transport_qp *qp = entry->qp;
1241 struct ntb_payload_header __iomem *hdr = entry->tx_hdr; 1329 struct ntb_payload_header __iomem *hdr = entry->tx_hdr;
1242 1330
1243 /* Ensure that the data is fully copied out before setting the flags */
1244 wmb();
1245 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags); 1331 iowrite32(entry->flags | DESC_DONE_FLAG, &hdr->flags);
1246 1332
1247 ntb_ring_doorbell(qp->ndev, qp->qp_num); 1333 ntb_peer_db_set(qp->ndev, BIT_ULL(qp->qp_num));
1248 1334
1249 /* The entry length can only be zero if the packet is intended to be a 1335 /* The entry length can only be zero if the packet is intended to be a
1250 * "link down" or similar. Since no payload is being sent in these 1336 * "link down" or similar. Since no payload is being sent in these
@@ -1265,6 +1351,9 @@ static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset)
1265{ 1351{
1266 memcpy_toio(offset, entry->buf, entry->len); 1352 memcpy_toio(offset, entry->buf, entry->len);
1267 1353
1354 /* Ensure that the data is fully copied out before setting the flags */
1355 wmb();
1356
1268 ntb_tx_copy_callback(entry); 1357 ntb_tx_copy_callback(entry);
1269} 1358}
1270 1359
@@ -1288,7 +1377,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
1288 entry->tx_hdr = hdr; 1377 entry->tx_hdr = hdr;
1289 1378
1290 iowrite32(entry->len, &hdr->len); 1379 iowrite32(entry->len, &hdr->len);
1291 iowrite32((u32) qp->tx_pkts, &hdr->ver); 1380 iowrite32((u32)qp->tx_pkts, &hdr->ver);
1292 1381
1293 if (!chan) 1382 if (!chan)
1294 goto err; 1383 goto err;
@@ -1298,8 +1387,8 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
1298 1387
1299 device = chan->device; 1388 device = chan->device;
1300 dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index; 1389 dest = qp->tx_mw_phys + qp->tx_max_frame * qp->tx_index;
1301 buff_off = (size_t) buf & ~PAGE_MASK; 1390 buff_off = (size_t)buf & ~PAGE_MASK;
1302 dest_off = (size_t) dest & ~PAGE_MASK; 1391 dest_off = (size_t)dest & ~PAGE_MASK;
1303 1392
1304 if (!is_dma_copy_aligned(device, buff_off, dest_off, len)) 1393 if (!is_dma_copy_aligned(device, buff_off, dest_off, len))
1305 goto err; 1394 goto err;
@@ -1347,9 +1436,6 @@ err:
1347static int ntb_process_tx(struct ntb_transport_qp *qp, 1436static int ntb_process_tx(struct ntb_transport_qp *qp,
1348 struct ntb_queue_entry *entry) 1437 struct ntb_queue_entry *entry)
1349{ 1438{
1350 dev_dbg(&ntb_query_pdev(qp->ndev)->dev, "%lld - tx %u, entry len %d flags %x buff %p\n",
1351 qp->tx_pkts, qp->tx_index, entry->len, entry->flags,
1352 entry->buf);
1353 if (qp->tx_index == qp->remote_rx_info->entry) { 1439 if (qp->tx_index == qp->remote_rx_info->entry) {
1354 qp->tx_ring_full++; 1440 qp->tx_ring_full++;
1355 return -EAGAIN; 1441 return -EAGAIN;
@@ -1376,14 +1462,14 @@ static int ntb_process_tx(struct ntb_transport_qp *qp,
1376 1462
1377static void ntb_send_link_down(struct ntb_transport_qp *qp) 1463static void ntb_send_link_down(struct ntb_transport_qp *qp)
1378{ 1464{
1379 struct pci_dev *pdev = ntb_query_pdev(qp->ndev); 1465 struct pci_dev *pdev = qp->ndev->pdev;
1380 struct ntb_queue_entry *entry; 1466 struct ntb_queue_entry *entry;
1381 int i, rc; 1467 int i, rc;
1382 1468
1383 if (qp->qp_link == NTB_LINK_DOWN) 1469 if (!qp->link_is_up)
1384 return; 1470 return;
1385 1471
1386 qp->qp_link = NTB_LINK_DOWN; 1472 qp->link_is_up = false;
1387 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num); 1473 dev_info(&pdev->dev, "qp %d: Link Down\n", qp->qp_num);
1388 1474
1389 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) { 1475 for (i = 0; i < NTB_LINK_DOWN_TIMEOUT; i++) {
@@ -1422,18 +1508,21 @@ static void ntb_send_link_down(struct ntb_transport_qp *qp)
1422 * RETURNS: pointer to newly created ntb_queue, NULL on error. 1508 * RETURNS: pointer to newly created ntb_queue, NULL on error.
1423 */ 1509 */
1424struct ntb_transport_qp * 1510struct ntb_transport_qp *
1425ntb_transport_create_queue(void *data, struct pci_dev *pdev, 1511ntb_transport_create_queue(void *data, struct device *client_dev,
1426 const struct ntb_queue_handlers *handlers) 1512 const struct ntb_queue_handlers *handlers)
1427{ 1513{
1514 struct ntb_dev *ndev;
1515 struct pci_dev *pdev;
1516 struct ntb_transport_ctx *nt;
1428 struct ntb_queue_entry *entry; 1517 struct ntb_queue_entry *entry;
1429 struct ntb_transport_qp *qp; 1518 struct ntb_transport_qp *qp;
1430 struct ntb_transport *nt; 1519 u64 qp_bit;
1431 unsigned int free_queue; 1520 unsigned int free_queue;
1432 int rc, i; 1521 int i;
1433 1522
1434 nt = ntb_find_transport(pdev); 1523 ndev = dev_ntb(client_dev->parent);
1435 if (!nt) 1524 pdev = ndev->pdev;
1436 goto err; 1525 nt = ndev->ctx;
1437 1526
1438 free_queue = ffs(nt->qp_bitmap); 1527 free_queue = ffs(nt->qp_bitmap);
1439 if (!free_queue) 1528 if (!free_queue)
@@ -1442,9 +1531,11 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev,
1442 /* decrement free_queue to make it zero based */ 1531 /* decrement free_queue to make it zero based */
1443 free_queue--; 1532 free_queue--;
1444 1533
1445 clear_bit(free_queue, &nt->qp_bitmap); 1534 qp = &nt->qp_vec[free_queue];
1535 qp_bit = BIT_ULL(qp->qp_num);
1536
1537 nt->qp_bitmap_free &= ~qp_bit;
1446 1538
1447 qp = &nt->qps[free_queue];
1448 qp->cb_data = data; 1539 qp->cb_data = data;
1449 qp->rx_handler = handlers->rx_handler; 1540 qp->rx_handler = handlers->rx_handler;
1450 qp->tx_handler = handlers->tx_handler; 1541 qp->tx_handler = handlers->tx_handler;
@@ -1458,7 +1549,7 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev,
1458 } 1549 }
1459 1550
1460 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1551 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
1461 entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); 1552 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1462 if (!entry) 1553 if (!entry)
1463 goto err1; 1554 goto err1;
1464 1555
@@ -1468,7 +1559,7 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev,
1468 } 1559 }
1469 1560
1470 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { 1561 for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
1471 entry = kzalloc(sizeof(struct ntb_queue_entry), GFP_ATOMIC); 1562 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1472 if (!entry) 1563 if (!entry)
1473 goto err2; 1564 goto err2;
1474 1565
@@ -1477,10 +1568,8 @@ ntb_transport_create_queue(void *data, struct pci_dev *pdev,
1477 &qp->tx_free_q); 1568 &qp->tx_free_q);
1478 } 1569 }
1479 1570
1480 rc = ntb_register_db_callback(qp->ndev, free_queue, qp, 1571 ntb_db_clear(qp->ndev, qp_bit);
1481 ntb_transport_rxc_db); 1572 ntb_db_clear_mask(qp->ndev, qp_bit);
1482 if (rc)
1483 goto err2;
1484 1573
1485 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num); 1574 dev_info(&pdev->dev, "NTB Transport QP %d created\n", qp->qp_num);
1486 1575
@@ -1494,7 +1583,7 @@ err1:
1494 kfree(entry); 1583 kfree(entry);
1495 if (qp->dma_chan) 1584 if (qp->dma_chan)
1496 dmaengine_put(); 1585 dmaengine_put();
1497 set_bit(free_queue, &nt->qp_bitmap); 1586 nt->qp_bitmap_free |= qp_bit;
1498err: 1587err:
1499 return NULL; 1588 return NULL;
1500} 1589}
@@ -1508,13 +1597,15 @@ EXPORT_SYMBOL_GPL(ntb_transport_create_queue);
1508 */ 1597 */
1509void ntb_transport_free_queue(struct ntb_transport_qp *qp) 1598void ntb_transport_free_queue(struct ntb_transport_qp *qp)
1510{ 1599{
1600 struct ntb_transport_ctx *nt = qp->transport;
1511 struct pci_dev *pdev; 1601 struct pci_dev *pdev;
1512 struct ntb_queue_entry *entry; 1602 struct ntb_queue_entry *entry;
1603 u64 qp_bit;
1513 1604
1514 if (!qp) 1605 if (!qp)
1515 return; 1606 return;
1516 1607
1517 pdev = ntb_query_pdev(qp->ndev); 1608 pdev = qp->ndev->pdev;
1518 1609
1519 if (qp->dma_chan) { 1610 if (qp->dma_chan) {
1520 struct dma_chan *chan = qp->dma_chan; 1611 struct dma_chan *chan = qp->dma_chan;
@@ -1531,10 +1622,18 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
1531 dmaengine_put(); 1622 dmaengine_put();
1532 } 1623 }
1533 1624
1534 ntb_unregister_db_callback(qp->ndev, qp->qp_num); 1625 qp_bit = BIT_ULL(qp->qp_num);
1626
1627 ntb_db_set_mask(qp->ndev, qp_bit);
1628 tasklet_disable(&qp->rxc_db_work);
1535 1629
1536 cancel_delayed_work_sync(&qp->link_work); 1630 cancel_delayed_work_sync(&qp->link_work);
1537 1631
1632 qp->cb_data = NULL;
1633 qp->rx_handler = NULL;
1634 qp->tx_handler = NULL;
1635 qp->event_handler = NULL;
1636
1538 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) 1637 while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
1539 kfree(entry); 1638 kfree(entry);
1540 1639
@@ -1546,7 +1645,7 @@ void ntb_transport_free_queue(struct ntb_transport_qp *qp)
1546 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) 1645 while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
1547 kfree(entry); 1646 kfree(entry);
1548 1647
1549 set_bit(qp->qp_num, &qp->transport->qp_bitmap); 1648 nt->qp_bitmap_free |= qp_bit;
1550 1649
1551 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num); 1650 dev_info(&pdev->dev, "NTB Transport QP %d freed\n", qp->qp_num);
1552} 1651}
@@ -1567,7 +1666,7 @@ void *ntb_transport_rx_remove(struct ntb_transport_qp *qp, unsigned int *len)
1567 struct ntb_queue_entry *entry; 1666 struct ntb_queue_entry *entry;
1568 void *buf; 1667 void *buf;
1569 1668
1570 if (!qp || qp->client_ready == NTB_LINK_UP) 1669 if (!qp || qp->client_ready)
1571 return NULL; 1670 return NULL;
1572 1671
1573 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q); 1672 entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
@@ -1636,7 +1735,7 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
1636 struct ntb_queue_entry *entry; 1735 struct ntb_queue_entry *entry;
1637 int rc; 1736 int rc;
1638 1737
1639 if (!qp || qp->qp_link != NTB_LINK_UP || !len) 1738 if (!qp || !qp->link_is_up || !len)
1640 return -EINVAL; 1739 return -EINVAL;
1641 1740
1642 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); 1741 entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q);
@@ -1670,9 +1769,9 @@ void ntb_transport_link_up(struct ntb_transport_qp *qp)
1670 if (!qp) 1769 if (!qp)
1671 return; 1770 return;
1672 1771
1673 qp->client_ready = NTB_LINK_UP; 1772 qp->client_ready = true;
1674 1773
1675 if (qp->transport->transport_link == NTB_LINK_UP) 1774 if (qp->transport->link_is_up)
1676 schedule_delayed_work(&qp->link_work, 0); 1775 schedule_delayed_work(&qp->link_work, 0);
1677} 1776}
1678EXPORT_SYMBOL_GPL(ntb_transport_link_up); 1777EXPORT_SYMBOL_GPL(ntb_transport_link_up);
@@ -1688,27 +1787,20 @@ EXPORT_SYMBOL_GPL(ntb_transport_link_up);
1688void ntb_transport_link_down(struct ntb_transport_qp *qp) 1787void ntb_transport_link_down(struct ntb_transport_qp *qp)
1689{ 1788{
1690 struct pci_dev *pdev; 1789 struct pci_dev *pdev;
1691 int rc, val; 1790 int val;
1692 1791
1693 if (!qp) 1792 if (!qp)
1694 return; 1793 return;
1695 1794
1696 pdev = ntb_query_pdev(qp->ndev); 1795 pdev = qp->ndev->pdev;
1697 qp->client_ready = NTB_LINK_DOWN; 1796 qp->client_ready = false;
1698 1797
1699 rc = ntb_read_local_spad(qp->ndev, QP_LINKS, &val); 1798 val = ntb_spad_read(qp->ndev, QP_LINKS);
1700 if (rc) {
1701 dev_err(&pdev->dev, "Error reading spad %d\n", QP_LINKS);
1702 return;
1703 }
1704 1799
1705 rc = ntb_write_remote_spad(qp->ndev, QP_LINKS, 1800 ntb_peer_spad_write(qp->ndev, QP_LINKS,
1706 val & ~(1 << qp->qp_num)); 1801 val & ~BIT(qp->qp_num));
1707 if (rc)
1708 dev_err(&pdev->dev, "Error writing %x to remote spad %d\n",
1709 val & ~(1 << qp->qp_num), QP_LINKS);
1710 1802
1711 if (qp->qp_link == NTB_LINK_UP) 1803 if (qp->link_is_up)
1712 ntb_send_link_down(qp); 1804 ntb_send_link_down(qp);
1713 else 1805 else
1714 cancel_delayed_work_sync(&qp->link_work); 1806 cancel_delayed_work_sync(&qp->link_work);
@@ -1728,7 +1820,7 @@ bool ntb_transport_link_query(struct ntb_transport_qp *qp)
1728 if (!qp) 1820 if (!qp)
1729 return false; 1821 return false;
1730 1822
1731 return qp->qp_link == NTB_LINK_UP; 1823 return qp->link_is_up;
1732} 1824}
1733EXPORT_SYMBOL_GPL(ntb_transport_link_query); 1825EXPORT_SYMBOL_GPL(ntb_transport_link_query);
1734 1826
@@ -1774,3 +1866,69 @@ unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp)
1774 return max; 1866 return max;
1775} 1867}
1776EXPORT_SYMBOL_GPL(ntb_transport_max_size); 1868EXPORT_SYMBOL_GPL(ntb_transport_max_size);
1869
1870static void ntb_transport_doorbell_callback(void *data, int vector)
1871{
1872 struct ntb_transport_ctx *nt = data;
1873 struct ntb_transport_qp *qp;
1874 u64 db_bits;
1875 unsigned int qp_num;
1876
1877 db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free &
1878 ntb_db_vector_mask(nt->ndev, vector));
1879
1880 while (db_bits) {
1881 qp_num = __ffs(db_bits);
1882 qp = &nt->qp_vec[qp_num];
1883
1884 tasklet_schedule(&qp->rxc_db_work);
1885
1886 db_bits &= ~BIT_ULL(qp_num);
1887 }
1888}
1889
1890static const struct ntb_ctx_ops ntb_transport_ops = {
1891 .link_event = ntb_transport_event_callback,
1892 .db_event = ntb_transport_doorbell_callback,
1893};
1894
1895static struct ntb_client ntb_transport_client = {
1896 .ops = {
1897 .probe = ntb_transport_probe,
1898 .remove = ntb_transport_free,
1899 },
1900};
1901
1902static int __init ntb_transport_init(void)
1903{
1904 int rc;
1905
1906 if (debugfs_initialized())
1907 nt_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1908
1909 rc = bus_register(&ntb_transport_bus);
1910 if (rc)
1911 goto err_bus;
1912
1913 rc = ntb_register_client(&ntb_transport_client);
1914 if (rc)
1915 goto err_client;
1916
1917 return 0;
1918
1919err_client:
1920 bus_unregister(&ntb_transport_bus);
1921err_bus:
1922 debugfs_remove_recursive(nt_debugfs_dir);
1923 return rc;
1924}
1925module_init(ntb_transport_init);
1926
1927static void __exit ntb_transport_exit(void)
1928{
1929 debugfs_remove_recursive(nt_debugfs_dir);
1930
1931 ntb_unregister_client(&ntb_transport_client);
1932 bus_unregister(&ntb_transport_bus);
1933}
1934module_exit(ntb_transport_exit);
diff --git a/include/linux/ntb_transport.h b/include/linux/ntb_transport.h
index f78b64cc09d5..2862861366a5 100644
--- a/include/linux/ntb_transport.h
+++ b/include/linux/ntb_transport.h
@@ -5,6 +5,7 @@
5 * GPL LICENSE SUMMARY 5 * GPL LICENSE SUMMARY
6 * 6 *
7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
8 * 9 *
9 * This program is free software; you can redistribute it and/or modify 10 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as 11 * it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
13 * BSD LICENSE 14 * BSD LICENSE
14 * 15 *
15 * Copyright(c) 2012 Intel Corporation. All rights reserved. 16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
16 * 18 *
17 * Redistribution and use in source and binary forms, with or without 19 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions 20 * modification, are permitted provided that the following conditions
@@ -40,7 +42,7 @@
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 * 44 *
43 * Intel PCIe NTB Linux driver 45 * PCIe NTB Transport Linux driver
44 * 46 *
45 * Contact Information: 47 * Contact Information:
46 * Jon Mason <jon.mason@intel.com> 48 * Jon Mason <jon.mason@intel.com>
@@ -48,21 +50,16 @@
48 50
49struct ntb_transport_qp; 51struct ntb_transport_qp;
50 52
51struct ntb_client { 53struct ntb_transport_client {
52 struct device_driver driver; 54 struct device_driver driver;
53 int (*probe)(struct pci_dev *pdev); 55 int (*probe)(struct device *client_dev);
54 void (*remove)(struct pci_dev *pdev); 56 void (*remove)(struct device *client_dev);
55}; 57};
56 58
57enum { 59int ntb_transport_register_client(struct ntb_transport_client *drvr);
58 NTB_LINK_DOWN = 0, 60void ntb_transport_unregister_client(struct ntb_transport_client *drvr);
59 NTB_LINK_UP, 61int ntb_transport_register_client_dev(char *device_name);
60}; 62void ntb_transport_unregister_client_dev(char *device_name);
61
62int ntb_transport_register_client(struct ntb_client *drvr);
63void ntb_transport_unregister_client(struct ntb_client *drvr);
64int ntb_register_client_dev(char *device_name);
65void ntb_unregister_client_dev(char *device_name);
66 63
67struct ntb_queue_handlers { 64struct ntb_queue_handlers {
68 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, 65 void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data,
@@ -75,7 +72,7 @@ struct ntb_queue_handlers {
75unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp); 72unsigned char ntb_transport_qp_num(struct ntb_transport_qp *qp);
76unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); 73unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp);
77struct ntb_transport_qp * 74struct ntb_transport_qp *
78ntb_transport_create_queue(void *data, struct pci_dev *pdev, 75ntb_transport_create_queue(void *data, struct device *client_dev,
79 const struct ntb_queue_handlers *handlers); 76 const struct ntb_queue_handlers *handlers);
80void ntb_transport_free_queue(struct ntb_transport_qp *qp); 77void ntb_transport_free_queue(struct ntb_transport_qp *qp);
81int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, 78int ntb_transport_rx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data,
generated by cgit v1.2.2 (git 2.25.0) at 2025-04-11 05:48:23 -0400