aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--MAINTAINERS6
-rw-r--r--drivers/ntb/hw/Kconfig1
-rw-r--r--drivers/ntb/hw/Makefile1
-rw-r--r--drivers/ntb/hw/amd/Kconfig7
-rw-r--r--drivers/ntb/hw/amd/Makefile1
-rw-r--r--drivers/ntb/hw/amd/ntb_hw_amd.c1143
-rw-r--r--drivers/ntb/hw/amd/ntb_hw_amd.h217
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_intel.c4
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_intel.h8
-rw-r--r--drivers/ntb/ntb_transport.c50
-rw-r--r--drivers/ntb/test/Kconfig8
-rw-r--r--drivers/ntb/test/Makefile1
-rw-r--r--drivers/ntb/test/ntb_perf.c748
13 files changed, 2183 insertions, 12 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index fc1718500181..8f3f93c9571e 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -7702,6 +7702,12 @@ W: https://github.com/jonmason/ntb/wiki
7702T: git git://github.com/jonmason/ntb.git 7702T: git git://github.com/jonmason/ntb.git
7703F: drivers/ntb/hw/intel/ 7703F: drivers/ntb/hw/intel/
7704 7704
7705NTB AMD DRIVER
7706M: Xiangliang Yu <Xiangliang.Yu@amd.com>
7707L: linux-ntb@googlegroups.com
7708S: Supported
7709F: drivers/ntb/hw/amd/
7710
7705NTFS FILESYSTEM 7711NTFS FILESYSTEM
7706M: Anton Altaparmakov <anton@tuxera.com> 7712M: Anton Altaparmakov <anton@tuxera.com>
7707L: linux-ntfs-dev@lists.sourceforge.net 7713L: linux-ntfs-dev@lists.sourceforge.net
diff --git a/drivers/ntb/hw/Kconfig b/drivers/ntb/hw/Kconfig
index 4d5535c4cddf..7116472b4625 100644
--- a/drivers/ntb/hw/Kconfig
+++ b/drivers/ntb/hw/Kconfig
@@ -1 +1,2 @@
1source "drivers/ntb/hw/amd/Kconfig"
1source "drivers/ntb/hw/intel/Kconfig" 2source "drivers/ntb/hw/intel/Kconfig"
diff --git a/drivers/ntb/hw/Makefile b/drivers/ntb/hw/Makefile
index 175d7c92a569..532e0859b4a1 100644
--- a/drivers/ntb/hw/Makefile
+++ b/drivers/ntb/hw/Makefile
@@ -1 +1,2 @@
1obj-$(CONFIG_NTB_AMD) += amd/
1obj-$(CONFIG_NTB_INTEL) += intel/ 2obj-$(CONFIG_NTB_INTEL) += intel/
diff --git a/drivers/ntb/hw/amd/Kconfig b/drivers/ntb/hw/amd/Kconfig
new file mode 100644
index 000000000000..cfe903cd9514
--- /dev/null
+++ b/drivers/ntb/hw/amd/Kconfig
@@ -0,0 +1,7 @@
1config NTB_AMD
2 tristate "AMD Non-Transparent Bridge support"
3 depends on X86_64
4 help
5 This driver supports AMD NTB on capable Zeppelin hardware.
6
7 If unsure, say N.
diff --git a/drivers/ntb/hw/amd/Makefile b/drivers/ntb/hw/amd/Makefile
new file mode 100644
index 000000000000..ad54da917563
--- /dev/null
+++ b/drivers/ntb/hw/amd/Makefile
@@ -0,0 +1 @@
obj-$(CONFIG_NTB_AMD) += ntb_hw_amd.o
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
new file mode 100644
index 000000000000..588803ad6847
--- /dev/null
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -0,0 +1,1143 @@
1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * BSD LICENSE
14 *
15 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
26 * distribution.
27 * * Neither the name of AMD Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * AMD PCIe NTB Linux driver
44 *
45 * Contact Information:
46 * Xiangliang Yu <Xiangliang.Yu@amd.com>
47 */
48
49#include <linux/debugfs.h>
50#include <linux/delay.h>
51#include <linux/init.h>
52#include <linux/interrupt.h>
53#include <linux/module.h>
54#include <linux/acpi.h>
55#include <linux/pci.h>
56#include <linux/random.h>
57#include <linux/slab.h>
58#include <linux/ntb.h>
59
60#include "ntb_hw_amd.h"
61
62#define NTB_NAME "ntb_hw_amd"
63#define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
64#define NTB_VER "1.0"
65
66MODULE_DESCRIPTION(NTB_DESC);
67MODULE_VERSION(NTB_VER);
68MODULE_LICENSE("Dual BSD/GPL");
69MODULE_AUTHOR("AMD Inc.");
70
71static const struct file_operations amd_ntb_debugfs_info;
72static struct dentry *debugfs_dir;
73
74static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
75{
76 if (idx < 0 || idx > ndev->mw_count)
77 return -EINVAL;
78
79 return 1 << idx;
80}
81
82static int amd_ntb_mw_count(struct ntb_dev *ntb)
83{
84 return ntb_ndev(ntb)->mw_count;
85}
86
87static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
88 phys_addr_t *base,
89 resource_size_t *size,
90 resource_size_t *align,
91 resource_size_t *align_size)
92{
93 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
94 int bar;
95
96 bar = ndev_mw_to_bar(ndev, idx);
97 if (bar < 0)
98 return bar;
99
100 if (base)
101 *base = pci_resource_start(ndev->ntb.pdev, bar);
102
103 if (size)
104 *size = pci_resource_len(ndev->ntb.pdev, bar);
105
106 if (align)
107 *align = SZ_4K;
108
109 if (align_size)
110 *align_size = 1;
111
112 return 0;
113}
114
115static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
116 dma_addr_t addr, resource_size_t size)
117{
118 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
119 unsigned long xlat_reg, limit_reg = 0;
120 resource_size_t mw_size;
121 void __iomem *mmio, *peer_mmio;
122 u64 base_addr, limit, reg_val;
123 int bar;
124
125 bar = ndev_mw_to_bar(ndev, idx);
126 if (bar < 0)
127 return bar;
128
129 mw_size = pci_resource_len(ndev->ntb.pdev, bar);
130
131 /* make sure the range fits in the usable mw size */
132 if (size > mw_size)
133 return -EINVAL;
134
135 mmio = ndev->self_mmio;
136 peer_mmio = ndev->peer_mmio;
137
138 base_addr = pci_resource_start(ndev->ntb.pdev, bar);
139
140 if (bar != 1) {
141 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 3);
142 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 3);
143
144 /* Set the limit if supported */
145 limit = base_addr + size;
146
147 /* set and verify setting the translation address */
148 write64(addr, peer_mmio + xlat_reg);
149 reg_val = read64(peer_mmio + xlat_reg);
150 if (reg_val != addr) {
151 write64(0, peer_mmio + xlat_reg);
152 return -EIO;
153 }
154
155 /* set and verify setting the limit */
156 write64(limit, mmio + limit_reg);
157 reg_val = read64(mmio + limit_reg);
158 if (reg_val != limit) {
159 write64(base_addr, mmio + limit_reg);
160 write64(0, peer_mmio + xlat_reg);
161 return -EIO;
162 }
163 } else {
164 xlat_reg = AMD_BAR1XLAT_OFFSET;
165 limit_reg = AMD_BAR1LMT_OFFSET;
166
167 /* split bar addr range must all be 32 bit */
168 if (addr & (~0ull << 32))
169 return -EINVAL;
170 if ((addr + size) & (~0ull << 32))
171 return -EINVAL;
172
173 /* Set the limit if supported */
174 limit = base_addr + size;
175
176 /* set and verify setting the translation address */
177 write64(addr, peer_mmio + xlat_reg);
178 reg_val = read64(peer_mmio + xlat_reg);
179 if (reg_val != addr) {
180 write64(0, peer_mmio + xlat_reg);
181 return -EIO;
182 }
183
184 /* set and verify setting the limit */
185 writel(limit, mmio + limit_reg);
186 reg_val = readl(mmio + limit_reg);
187 if (reg_val != limit) {
188 writel(base_addr, mmio + limit_reg);
189 writel(0, peer_mmio + xlat_reg);
190 return -EIO;
191 }
192 }
193
194 return 0;
195}
196
197static int amd_link_is_up(struct amd_ntb_dev *ndev)
198{
199 if (!ndev->peer_sta)
200 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
201
202 /* If peer_sta is reset or D0 event, the ISR has
203 * started a timer to check link status of hardware.
204 * So here just clear status bit. And if peer_sta is
205 * D3 or PME_TO, D0/reset event will be happened when
206 * system wakeup/poweron, so do nothing here.
207 */
208 if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
209 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
210 else if (ndev->peer_sta & AMD_PEER_D0_EVENT)
211 ndev->peer_sta = 0;
212
213 return 0;
214}
215
216static int amd_ntb_link_is_up(struct ntb_dev *ntb,
217 enum ntb_speed *speed,
218 enum ntb_width *width)
219{
220 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
221 int ret = 0;
222
223 if (amd_link_is_up(ndev)) {
224 if (speed)
225 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
226 if (width)
227 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
228
229 dev_dbg(ndev_dev(ndev), "link is up.\n");
230
231 ret = 1;
232 } else {
233 if (speed)
234 *speed = NTB_SPEED_NONE;
235 if (width)
236 *width = NTB_WIDTH_NONE;
237
238 dev_dbg(ndev_dev(ndev), "link is down.\n");
239 }
240
241 return ret;
242}
243
244static int amd_ntb_link_enable(struct ntb_dev *ntb,
245 enum ntb_speed max_speed,
246 enum ntb_width max_width)
247{
248 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
249 void __iomem *mmio = ndev->self_mmio;
250 u32 ntb_ctl;
251
252 /* Enable event interrupt */
253 ndev->int_mask &= ~AMD_EVENT_INTMASK;
254 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
255
256 if (ndev->ntb.topo == NTB_TOPO_SEC)
257 return -EINVAL;
258 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
259
260 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
261 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
262 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
263
264 return 0;
265}
266
267static int amd_ntb_link_disable(struct ntb_dev *ntb)
268{
269 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
270 void __iomem *mmio = ndev->self_mmio;
271 u32 ntb_ctl;
272
273 /* Disable event interrupt */
274 ndev->int_mask |= AMD_EVENT_INTMASK;
275 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
276
277 if (ndev->ntb.topo == NTB_TOPO_SEC)
278 return -EINVAL;
279 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
280
281 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
282 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
283 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
284
285 return 0;
286}
287
288static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
289{
290 return ntb_ndev(ntb)->db_valid_mask;
291}
292
293static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
294{
295 return ntb_ndev(ntb)->db_count;
296}
297
298static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
299{
300 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
301
302 if (db_vector < 0 || db_vector > ndev->db_count)
303 return 0;
304
305 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
306}
307
308static u64 amd_ntb_db_read(struct ntb_dev *ntb)
309{
310 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
311 void __iomem *mmio = ndev->self_mmio;
312
313 return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
314}
315
316static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
317{
318 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
319 void __iomem *mmio = ndev->self_mmio;
320
321 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
322
323 return 0;
324}
325
326static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
327{
328 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
329 void __iomem *mmio = ndev->self_mmio;
330 unsigned long flags;
331
332 if (db_bits & ~ndev->db_valid_mask)
333 return -EINVAL;
334
335 spin_lock_irqsave(&ndev->db_mask_lock, flags);
336 ndev->db_mask |= db_bits;
337 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
338 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
339
340 return 0;
341}
342
343static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
344{
345 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
346 void __iomem *mmio = ndev->self_mmio;
347 unsigned long flags;
348
349 if (db_bits & ~ndev->db_valid_mask)
350 return -EINVAL;
351
352 spin_lock_irqsave(&ndev->db_mask_lock, flags);
353 ndev->db_mask &= ~db_bits;
354 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
355 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
356
357 return 0;
358}
359
360static int amd_ntb_peer_db_addr(struct ntb_dev *ntb,
361 phys_addr_t *db_addr,
362 resource_size_t *db_size)
363{
364 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
365
366 if (db_addr)
367 *db_addr = (phys_addr_t)(ndev->peer_mmio + AMD_DBREQ_OFFSET);
368 if (db_size)
369 *db_size = sizeof(u32);
370
371 return 0;
372}
373
374static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
375{
376 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
377 void __iomem *mmio = ndev->self_mmio;
378
379 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
380
381 return 0;
382}
383
384static int amd_ntb_spad_count(struct ntb_dev *ntb)
385{
386 return ntb_ndev(ntb)->spad_count;
387}
388
389static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
390{
391 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
392 void __iomem *mmio = ndev->self_mmio;
393 u32 offset;
394
395 if (idx < 0 || idx >= ndev->spad_count)
396 return 0;
397
398 offset = ndev->self_spad + (idx << 2);
399 return readl(mmio + AMD_SPAD_OFFSET + offset);
400}
401
402static int amd_ntb_spad_write(struct ntb_dev *ntb,
403 int idx, u32 val)
404{
405 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
406 void __iomem *mmio = ndev->self_mmio;
407 u32 offset;
408
409 if (idx < 0 || idx >= ndev->spad_count)
410 return -EINVAL;
411
412 offset = ndev->self_spad + (idx << 2);
413 writel(val, mmio + AMD_SPAD_OFFSET + offset);
414
415 return 0;
416}
417
418static int amd_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
419 phys_addr_t *spad_addr)
420{
421 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
422
423 if (idx < 0 || idx >= ndev->spad_count)
424 return -EINVAL;
425
426 if (spad_addr)
427 *spad_addr = (phys_addr_t)(ndev->self_mmio + AMD_SPAD_OFFSET +
428 ndev->peer_spad + (idx << 2));
429 return 0;
430}
431
432static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
433{
434 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
435 void __iomem *mmio = ndev->self_mmio;
436 u32 offset;
437
438 if (idx < 0 || idx >= ndev->spad_count)
439 return -EINVAL;
440
441 offset = ndev->peer_spad + (idx << 2);
442 return readl(mmio + AMD_SPAD_OFFSET + offset);
443}
444
445static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
446 int idx, u32 val)
447{
448 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
449 void __iomem *mmio = ndev->self_mmio;
450 u32 offset;
451
452 if (idx < 0 || idx >= ndev->spad_count)
453 return -EINVAL;
454
455 offset = ndev->peer_spad + (idx << 2);
456 writel(val, mmio + AMD_SPAD_OFFSET + offset);
457
458 return 0;
459}
460
461static const struct ntb_dev_ops amd_ntb_ops = {
462 .mw_count = amd_ntb_mw_count,
463 .mw_get_range = amd_ntb_mw_get_range,
464 .mw_set_trans = amd_ntb_mw_set_trans,
465 .link_is_up = amd_ntb_link_is_up,
466 .link_enable = amd_ntb_link_enable,
467 .link_disable = amd_ntb_link_disable,
468 .db_valid_mask = amd_ntb_db_valid_mask,
469 .db_vector_count = amd_ntb_db_vector_count,
470 .db_vector_mask = amd_ntb_db_vector_mask,
471 .db_read = amd_ntb_db_read,
472 .db_clear = amd_ntb_db_clear,
473 .db_set_mask = amd_ntb_db_set_mask,
474 .db_clear_mask = amd_ntb_db_clear_mask,
475 .peer_db_addr = amd_ntb_peer_db_addr,
476 .peer_db_set = amd_ntb_peer_db_set,
477 .spad_count = amd_ntb_spad_count,
478 .spad_read = amd_ntb_spad_read,
479 .spad_write = amd_ntb_spad_write,
480 .peer_spad_addr = amd_ntb_peer_spad_addr,
481 .peer_spad_read = amd_ntb_peer_spad_read,
482 .peer_spad_write = amd_ntb_peer_spad_write,
483};
484
485static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
486{
487 void __iomem *mmio = ndev->self_mmio;
488 int reg;
489
490 reg = readl(mmio + AMD_SMUACK_OFFSET);
491 reg |= bit;
492 writel(reg, mmio + AMD_SMUACK_OFFSET);
493
494 ndev->peer_sta |= bit;
495}
496
497static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
498{
499 void __iomem *mmio = ndev->self_mmio;
500 u32 status;
501
502 status = readl(mmio + AMD_INTSTAT_OFFSET);
503 if (!(status & AMD_EVENT_INTMASK))
504 return;
505
506 dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec);
507
508 status &= AMD_EVENT_INTMASK;
509 switch (status) {
510 case AMD_PEER_FLUSH_EVENT:
511 dev_info(ndev_dev(ndev), "Flush is done.\n");
512 break;
513 case AMD_PEER_RESET_EVENT:
514 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
515
516 /* link down first */
517 ntb_link_event(&ndev->ntb);
518 /* polling peer status */
519 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
520
521 break;
522 case AMD_PEER_D3_EVENT:
523 case AMD_PEER_PMETO_EVENT:
524 amd_ack_smu(ndev, status);
525
526 /* link down */
527 ntb_link_event(&ndev->ntb);
528
529 break;
530 case AMD_PEER_D0_EVENT:
531 mmio = ndev->peer_mmio;
532 status = readl(mmio + AMD_PMESTAT_OFFSET);
533 /* check if this is WAKEUP event */
534 if (status & 0x1)
535 dev_info(ndev_dev(ndev), "Wakeup is done.\n");
536
537 amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
538
539 /* start a timer to poll link status */
540 schedule_delayed_work(&ndev->hb_timer,
541 AMD_LINK_HB_TIMEOUT);
542 break;
543 default:
544 dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status);
545 break;
546 }
547}
548
549static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
550{
551 dev_dbg(ndev_dev(ndev), "vec %d\n", vec);
552
553 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
554 amd_handle_event(ndev, vec);
555
556 if (vec < AMD_DB_CNT)
557 ntb_db_event(&ndev->ntb, vec);
558
559 return IRQ_HANDLED;
560}
561
562static irqreturn_t ndev_vec_isr(int irq, void *dev)
563{
564 struct amd_ntb_vec *nvec = dev;
565
566 return ndev_interrupt(nvec->ndev, nvec->num);
567}
568
569static irqreturn_t ndev_irq_isr(int irq, void *dev)
570{
571 struct amd_ntb_dev *ndev = dev;
572
573 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
574}
575
576static int ndev_init_isr(struct amd_ntb_dev *ndev,
577 int msix_min, int msix_max)
578{
579 struct pci_dev *pdev;
580 int rc, i, msix_count, node;
581
582 pdev = ndev_pdev(ndev);
583
584 node = dev_to_node(&pdev->dev);
585
586 ndev->db_mask = ndev->db_valid_mask;
587
588 /* Try to set up msix irq */
589 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
590 GFP_KERNEL, node);
591 if (!ndev->vec)
592 goto err_msix_vec_alloc;
593
594 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
595 GFP_KERNEL, node);
596 if (!ndev->msix)
597 goto err_msix_alloc;
598
599 for (i = 0; i < msix_max; ++i)
600 ndev->msix[i].entry = i;
601
602 msix_count = pci_enable_msix_range(pdev, ndev->msix,
603 msix_min, msix_max);
604 if (msix_count < 0)
605 goto err_msix_enable;
606
607 /* NOTE: Disable MSIX if msix count is less than 16 because of
608 * hardware limitation.
609 */
610 if (msix_count < msix_min) {
611 pci_disable_msix(pdev);
612 goto err_msix_enable;
613 }
614
615 for (i = 0; i < msix_count; ++i) {
616 ndev->vec[i].ndev = ndev;
617 ndev->vec[i].num = i;
618 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
619 "ndev_vec_isr", &ndev->vec[i]);
620 if (rc)
621 goto err_msix_request;
622 }
623
624 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
625 ndev->db_count = msix_min;
626 ndev->msix_vec_count = msix_max;
627 return 0;
628
629err_msix_request:
630 while (i-- > 0)
631 free_irq(ndev->msix[i].vector, ndev);
632 pci_disable_msix(pdev);
633err_msix_enable:
634 kfree(ndev->msix);
635err_msix_alloc:
636 kfree(ndev->vec);
637err_msix_vec_alloc:
638 ndev->msix = NULL;
639 ndev->vec = NULL;
640
641 /* Try to set up msi irq */
642 rc = pci_enable_msi(pdev);
643 if (rc)
644 goto err_msi_enable;
645
646 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
647 "ndev_irq_isr", ndev);
648 if (rc)
649 goto err_msi_request;
650
651 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
652 ndev->db_count = 1;
653 ndev->msix_vec_count = 1;
654 return 0;
655
656err_msi_request:
657 pci_disable_msi(pdev);
658err_msi_enable:
659
660 /* Try to set up intx irq */
661 pci_intx(pdev, 1);
662
663 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
664 "ndev_irq_isr", ndev);
665 if (rc)
666 goto err_intx_request;
667
668 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
669 ndev->db_count = 1;
670 ndev->msix_vec_count = 1;
671 return 0;
672
673err_intx_request:
674 return rc;
675}
676
677static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
678{
679 struct pci_dev *pdev;
680 void __iomem *mmio = ndev->self_mmio;
681 int i;
682
683 pdev = ndev_pdev(ndev);
684
685 /* Mask all doorbell interrupts */
686 ndev->db_mask = ndev->db_valid_mask;
687 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
688
689 if (ndev->msix) {
690 i = ndev->msix_vec_count;
691 while (i--)
692 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
693 pci_disable_msix(pdev);
694 kfree(ndev->msix);
695 kfree(ndev->vec);
696 } else {
697 free_irq(pdev->irq, ndev);
698 if (pci_dev_msi_enabled(pdev))
699 pci_disable_msi(pdev);
700 else
701 pci_intx(pdev, 0);
702 }
703}
704
705static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
706 size_t count, loff_t *offp)
707{
708 struct amd_ntb_dev *ndev;
709 void __iomem *mmio;
710 char *buf;
711 size_t buf_size;
712 ssize_t ret, off;
713 union { u64 v64; u32 v32; u16 v16; } u;
714
715 ndev = filp->private_data;
716 mmio = ndev->self_mmio;
717
718 buf_size = min(count, 0x800ul);
719
720 buf = kmalloc(buf_size, GFP_KERNEL);
721 if (!buf)
722 return -ENOMEM;
723
724 off = 0;
725
726 off += scnprintf(buf + off, buf_size - off,
727 "NTB Device Information:\n");
728
729 off += scnprintf(buf + off, buf_size - off,
730 "Connection Topology -\t%s\n",
731 ntb_topo_string(ndev->ntb.topo));
732
733 off += scnprintf(buf + off, buf_size - off,
734 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
735
736 if (!amd_link_is_up(ndev)) {
737 off += scnprintf(buf + off, buf_size - off,
738 "Link Status -\t\tDown\n");
739 } else {
740 off += scnprintf(buf + off, buf_size - off,
741 "Link Status -\t\tUp\n");
742 off += scnprintf(buf + off, buf_size - off,
743 "Link Speed -\t\tPCI-E Gen %u\n",
744 NTB_LNK_STA_SPEED(ndev->lnk_sta));
745 off += scnprintf(buf + off, buf_size - off,
746 "Link Width -\t\tx%u\n",
747 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
748 }
749
750 off += scnprintf(buf + off, buf_size - off,
751 "Memory Window Count -\t%u\n", ndev->mw_count);
752 off += scnprintf(buf + off, buf_size - off,
753 "Scratchpad Count -\t%u\n", ndev->spad_count);
754 off += scnprintf(buf + off, buf_size - off,
755 "Doorbell Count -\t%u\n", ndev->db_count);
756 off += scnprintf(buf + off, buf_size - off,
757 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
758
759 off += scnprintf(buf + off, buf_size - off,
760 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
761
762 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
763 off += scnprintf(buf + off, buf_size - off,
764 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
765
766 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
767 off += scnprintf(buf + off, buf_size - off,
768 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
769
770 off += scnprintf(buf + off, buf_size - off,
771 "\nNTB Incoming XLAT:\n");
772
773 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
774 off += scnprintf(buf + off, buf_size - off,
775 "XLAT1 -\t\t%#018llx\n", u.v64);
776
777 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
778 off += scnprintf(buf + off, buf_size - off,
779 "XLAT23 -\t\t%#018llx\n", u.v64);
780
781 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
782 off += scnprintf(buf + off, buf_size - off,
783 "XLAT45 -\t\t%#018llx\n", u.v64);
784
785 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
786 off += scnprintf(buf + off, buf_size - off,
787 "LMT1 -\t\t\t%#06x\n", u.v32);
788
789 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
790 off += scnprintf(buf + off, buf_size - off,
791 "LMT23 -\t\t\t%#018llx\n", u.v64);
792
793 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
794 off += scnprintf(buf + off, buf_size - off,
795 "LMT45 -\t\t\t%#018llx\n", u.v64);
796
797 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
798 kfree(buf);
799 return ret;
800}
801
802static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
803{
804 if (!debugfs_dir) {
805 ndev->debugfs_dir = NULL;
806 ndev->debugfs_info = NULL;
807 } else {
808 ndev->debugfs_dir =
809 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
810 if (!ndev->debugfs_dir)
811 ndev->debugfs_info = NULL;
812 else
813 ndev->debugfs_info =
814 debugfs_create_file("info", S_IRUSR,
815 ndev->debugfs_dir, ndev,
816 &amd_ntb_debugfs_info);
817 }
818}
819
820static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
821{
822 debugfs_remove_recursive(ndev->debugfs_dir);
823}
824
825static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
826 struct pci_dev *pdev)
827{
828 ndev->ntb.pdev = pdev;
829 ndev->ntb.topo = NTB_TOPO_NONE;
830 ndev->ntb.ops = &amd_ntb_ops;
831 ndev->int_mask = AMD_EVENT_INTMASK;
832 spin_lock_init(&ndev->db_mask_lock);
833}
834
835static int amd_poll_link(struct amd_ntb_dev *ndev)
836{
837 void __iomem *mmio = ndev->peer_mmio;
838 u32 reg, stat;
839 int rc;
840
841 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
842 reg &= NTB_LIN_STA_ACTIVE_BIT;
843
844 dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg);
845
846 if (reg == ndev->cntl_sta)
847 return 0;
848
849 ndev->cntl_sta = reg;
850
851 rc = pci_read_config_dword(ndev->ntb.pdev,
852 AMD_LINK_STATUS_OFFSET, &stat);
853 if (rc)
854 return 0;
855 ndev->lnk_sta = stat;
856
857 return 1;
858}
859
860static void amd_link_hb(struct work_struct *work)
861{
862 struct amd_ntb_dev *ndev = hb_ndev(work);
863
864 if (amd_poll_link(ndev))
865 ntb_link_event(&ndev->ntb);
866
867 if (!amd_link_is_up(ndev))
868 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
869}
870
871static int amd_init_isr(struct amd_ntb_dev *ndev)
872{
873 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
874}
875
876static void amd_init_side_info(struct amd_ntb_dev *ndev)
877{
878 void __iomem *mmio = ndev->self_mmio;
879 unsigned int reg;
880
881 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
882 if (!(reg & AMD_SIDE_READY)) {
883 reg |= AMD_SIDE_READY;
884 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
885 }
886}
887
888static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
889{
890 void __iomem *mmio = ndev->self_mmio;
891 unsigned int reg;
892
893 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
894 if (reg & AMD_SIDE_READY) {
895 reg &= ~AMD_SIDE_READY;
896 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
897 readl(mmio + AMD_SIDEINFO_OFFSET);
898 }
899}
900
901static int amd_init_ntb(struct amd_ntb_dev *ndev)
902{
903 void __iomem *mmio = ndev->self_mmio;
904
905 ndev->mw_count = AMD_MW_CNT;
906 ndev->spad_count = AMD_SPADS_CNT;
907 ndev->db_count = AMD_DB_CNT;
908
909 switch (ndev->ntb.topo) {
910 case NTB_TOPO_PRI:
911 case NTB_TOPO_SEC:
912 ndev->spad_count >>= 1;
913 if (ndev->ntb.topo == NTB_TOPO_PRI) {
914 ndev->self_spad = 0;
915 ndev->peer_spad = 0x20;
916 } else {
917 ndev->self_spad = 0x20;
918 ndev->peer_spad = 0;
919 }
920
921 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
922 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
923
924 break;
925 default:
926 dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n");
927 return -EINVAL;
928 }
929
930 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
931
932 /* Mask event interrupts */
933 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
934
935 return 0;
936}
937
938static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
939{
940 void __iomem *mmio = ndev->self_mmio;
941 u32 info;
942
943 info = readl(mmio + AMD_SIDEINFO_OFFSET);
944 if (info & AMD_SIDE_MASK)
945 return NTB_TOPO_SEC;
946 else
947 return NTB_TOPO_PRI;
948}
949
950static int amd_init_dev(struct amd_ntb_dev *ndev)
951{
952 struct pci_dev *pdev;
953 int rc = 0;
954
955 pdev = ndev_pdev(ndev);
956
957 ndev->ntb.topo = amd_get_topo(ndev);
958 dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n",
959 ntb_topo_string(ndev->ntb.topo));
960
961 rc = amd_init_ntb(ndev);
962 if (rc)
963 return rc;
964
965 rc = amd_init_isr(ndev);
966 if (rc) {
967 dev_err(ndev_dev(ndev), "fail to init isr.\n");
968 return rc;
969 }
970
971 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
972
973 return 0;
974}
975
976static void amd_deinit_dev(struct amd_ntb_dev *ndev)
977{
978 cancel_delayed_work_sync(&ndev->hb_timer);
979
980 ndev_deinit_isr(ndev);
981}
982
983static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
984 struct pci_dev *pdev)
985{
986 int rc;
987
988 pci_set_drvdata(pdev, ndev);
989
990 rc = pci_enable_device(pdev);
991 if (rc)
992 goto err_pci_enable;
993
994 rc = pci_request_regions(pdev, NTB_NAME);
995 if (rc)
996 goto err_pci_regions;
997
998 pci_set_master(pdev);
999
1000 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1001 if (rc) {
1002 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1003 if (rc)
1004 goto err_dma_mask;
1005 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
1006 }
1007
1008 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1009 if (rc) {
1010 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1011 if (rc)
1012 goto err_dma_mask;
1013 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
1014 }
1015
1016 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1017 if (!ndev->self_mmio) {
1018 rc = -EIO;
1019 goto err_dma_mask;
1020 }
1021 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
1022
1023 return 0;
1024
1025err_dma_mask:
1026 pci_clear_master(pdev);
1027err_pci_regions:
1028 pci_disable_device(pdev);
1029err_pci_enable:
1030 pci_set_drvdata(pdev, NULL);
1031 return rc;
1032}
1033
1034static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1035{
1036 struct pci_dev *pdev = ndev_pdev(ndev);
1037
1038 pci_iounmap(pdev, ndev->self_mmio);
1039
1040 pci_clear_master(pdev);
1041 pci_release_regions(pdev);
1042 pci_disable_device(pdev);
1043 pci_set_drvdata(pdev, NULL);
1044}
1045
1046static int amd_ntb_pci_probe(struct pci_dev *pdev,
1047 const struct pci_device_id *id)
1048{
1049 struct amd_ntb_dev *ndev;
1050 int rc, node;
1051
1052 node = dev_to_node(&pdev->dev);
1053
1054 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1055 if (!ndev) {
1056 rc = -ENOMEM;
1057 goto err_ndev;
1058 }
1059
1060 ndev_init_struct(ndev, pdev);
1061
1062 rc = amd_ntb_init_pci(ndev, pdev);
1063 if (rc)
1064 goto err_init_pci;
1065
1066 rc = amd_init_dev(ndev);
1067 if (rc)
1068 goto err_init_dev;
1069
1070 /* write side info */
1071 amd_init_side_info(ndev);
1072
1073 amd_poll_link(ndev);
1074
1075 ndev_init_debugfs(ndev);
1076
1077 rc = ntb_register_device(&ndev->ntb);
1078 if (rc)
1079 goto err_register;
1080
1081 dev_info(&pdev->dev, "NTB device registered.\n");
1082
1083 return 0;
1084
1085err_register:
1086 ndev_deinit_debugfs(ndev);
1087 amd_deinit_dev(ndev);
1088err_init_dev:
1089 amd_ntb_deinit_pci(ndev);
1090err_init_pci:
1091 kfree(ndev);
1092err_ndev:
1093 return rc;
1094}
1095
1096static void amd_ntb_pci_remove(struct pci_dev *pdev)
1097{
1098 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1099
1100 ntb_unregister_device(&ndev->ntb);
1101 ndev_deinit_debugfs(ndev);
1102 amd_deinit_side_info(ndev);
1103 amd_deinit_dev(ndev);
1104 amd_ntb_deinit_pci(ndev);
1105 kfree(ndev);
1106}
1107
1108static const struct file_operations amd_ntb_debugfs_info = {
1109 .owner = THIS_MODULE,
1110 .open = simple_open,
1111 .read = ndev_debugfs_read,
1112};
1113
1114static const struct pci_device_id amd_ntb_pci_tbl[] = {
1115 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)},
1116 {0}
1117};
1118MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1119
1120static struct pci_driver amd_ntb_pci_driver = {
1121 .name = KBUILD_MODNAME,
1122 .id_table = amd_ntb_pci_tbl,
1123 .probe = amd_ntb_pci_probe,
1124 .remove = amd_ntb_pci_remove,
1125};
1126
1127static int __init amd_ntb_pci_driver_init(void)
1128{
1129 pr_info("%s %s\n", NTB_DESC, NTB_VER);
1130
1131 if (debugfs_initialized())
1132 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1133
1134 return pci_register_driver(&amd_ntb_pci_driver);
1135}
1136module_init(amd_ntb_pci_driver_init);
1137
1138static void __exit amd_ntb_pci_driver_exit(void)
1139{
1140 pci_unregister_driver(&amd_ntb_pci_driver);
1141 debugfs_remove_recursive(debugfs_dir);
1142}
1143module_exit(amd_ntb_pci_driver_exit);
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.h b/drivers/ntb/hw/amd/ntb_hw_amd.h
new file mode 100644
index 000000000000..2eac3cd3e646
--- /dev/null
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.h
@@ -0,0 +1,217 @@
1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * BSD LICENSE
14 *
15 * Copyright (C) 2016 Advanced Micro Devices, Inc. All Rights Reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
26 * distribution.
27 * * Neither the name of AMD Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * AMD PCIe NTB Linux driver
44 *
45 * Contact Information:
46 * Xiangliang Yu <Xiangliang.Yu@amd.com>
47 */
48
49#ifndef NTB_HW_AMD_H
50#define NTB_HW_AMD_H
51
52#include <linux/ntb.h>
53#include <linux/pci.h>
54
55#define PCI_DEVICE_ID_AMD_NTB 0x145B
56#define AMD_LINK_HB_TIMEOUT msecs_to_jiffies(1000)
57#define AMD_LINK_STATUS_OFFSET 0x68
58#define NTB_LIN_STA_ACTIVE_BIT 0x00000002
59#define NTB_LNK_STA_SPEED_MASK 0x000F0000
60#define NTB_LNK_STA_WIDTH_MASK 0x03F00000
61#define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LIN_STA_ACTIVE_BIT))
62#define NTB_LNK_STA_SPEED(x) (((x) & NTB_LNK_STA_SPEED_MASK) >> 16)
63#define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 20)
64
65#ifndef read64
66#ifdef readq
67#define read64 readq
68#else
69#define read64 _read64
70static inline u64 _read64(void __iomem *mmio)
71{
72 u64 low, high;
73
74 low = readl(mmio);
75 high = readl(mmio + sizeof(u32));
76 return low | (high << 32);
77}
78#endif
79#endif
80
81#ifndef write64
82#ifdef writeq
83#define write64 writeq
84#else
85#define write64 _write64
86static inline void _write64(u64 val, void __iomem *mmio)
87{
88 writel(val, mmio);
89 writel(val >> 32, mmio + sizeof(u32));
90}
91#endif
92#endif
93
94enum {
95 /* AMD NTB Capability */
96 AMD_MW_CNT = 3,
97 AMD_DB_CNT = 16,
98 AMD_MSIX_VECTOR_CNT = 24,
99 AMD_SPADS_CNT = 16,
100
101 /* AMD NTB register offset */
102 AMD_CNTL_OFFSET = 0x200,
103
104 /* NTB control register bits */
105 PMM_REG_CTL = BIT(21),
106 SMM_REG_CTL = BIT(20),
107 SMM_REG_ACC_PATH = BIT(18),
108 PMM_REG_ACC_PATH = BIT(17),
109 NTB_CLK_EN = BIT(16),
110
111 AMD_STA_OFFSET = 0x204,
112 AMD_PGSLV_OFFSET = 0x208,
113 AMD_SPAD_MUX_OFFSET = 0x20C,
114 AMD_SPAD_OFFSET = 0x210,
115 AMD_RSMU_HCID = 0x250,
116 AMD_RSMU_SIID = 0x254,
117 AMD_PSION_OFFSET = 0x300,
118 AMD_SSION_OFFSET = 0x330,
119 AMD_MMINDEX_OFFSET = 0x400,
120 AMD_MMDATA_OFFSET = 0x404,
121 AMD_SIDEINFO_OFFSET = 0x408,
122
123 AMD_SIDE_MASK = BIT(0),
124 AMD_SIDE_READY = BIT(1),
125
126 /* limit register */
127 AMD_ROMBARLMT_OFFSET = 0x410,
128 AMD_BAR1LMT_OFFSET = 0x414,
129 AMD_BAR23LMT_OFFSET = 0x418,
130 AMD_BAR45LMT_OFFSET = 0x420,
131 /* xlat address */
132 AMD_POMBARXLAT_OFFSET = 0x428,
133 AMD_BAR1XLAT_OFFSET = 0x430,
134 AMD_BAR23XLAT_OFFSET = 0x438,
135 AMD_BAR45XLAT_OFFSET = 0x440,
136 /* doorbell and interrupt */
137 AMD_DBFM_OFFSET = 0x450,
138 AMD_DBREQ_OFFSET = 0x454,
139 AMD_MIRRDBSTAT_OFFSET = 0x458,
140 AMD_DBMASK_OFFSET = 0x45C,
141 AMD_DBSTAT_OFFSET = 0x460,
142 AMD_INTMASK_OFFSET = 0x470,
143 AMD_INTSTAT_OFFSET = 0x474,
144
145 /* event type */
146 AMD_PEER_FLUSH_EVENT = BIT(0),
147 AMD_PEER_RESET_EVENT = BIT(1),
148 AMD_PEER_D3_EVENT = BIT(2),
149 AMD_PEER_PMETO_EVENT = BIT(3),
150 AMD_PEER_D0_EVENT = BIT(4),
151 AMD_EVENT_INTMASK = (AMD_PEER_FLUSH_EVENT |
152 AMD_PEER_RESET_EVENT | AMD_PEER_D3_EVENT |
153 AMD_PEER_PMETO_EVENT | AMD_PEER_D0_EVENT),
154
155 AMD_PMESTAT_OFFSET = 0x480,
156 AMD_PMSGTRIG_OFFSET = 0x490,
157 AMD_LTRLATENCY_OFFSET = 0x494,
158 AMD_FLUSHTRIG_OFFSET = 0x498,
159
160 /* SMU register*/
161 AMD_SMUACK_OFFSET = 0x4A0,
162 AMD_SINRST_OFFSET = 0x4A4,
163 AMD_RSPNUM_OFFSET = 0x4A8,
164 AMD_SMU_SPADMUTEX = 0x4B0,
165 AMD_SMU_SPADOFFSET = 0x4B4,
166
167 AMD_PEER_OFFSET = 0x400,
168};
169
170struct amd_ntb_dev;
171
172struct amd_ntb_vec {
173 struct amd_ntb_dev *ndev;
174 int num;
175};
176
177struct amd_ntb_dev {
178 struct ntb_dev ntb;
179
180 u32 ntb_side;
181 u32 lnk_sta;
182 u32 cntl_sta;
183 u32 peer_sta;
184
185 unsigned char mw_count;
186 unsigned char spad_count;
187 unsigned char db_count;
188 unsigned char msix_vec_count;
189
190 u64 db_valid_mask;
191 u64 db_mask;
192 u32 int_mask;
193
194 struct msix_entry *msix;
195 struct amd_ntb_vec *vec;
196
197 /* synchronize rmw access of db_mask and hw reg */
198 spinlock_t db_mask_lock;
199
200 void __iomem *self_mmio;
201 void __iomem *peer_mmio;
202 unsigned int self_spad;
203 unsigned int peer_spad;
204
205 struct delayed_work hb_timer;
206
207 struct dentry *debugfs_dir;
208 struct dentry *debugfs_info;
209};
210
211#define ndev_pdev(ndev) ((ndev)->ntb.pdev)
212#define ndev_name(ndev) pci_name(ndev_pdev(ndev))
213#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
214#define ntb_ndev(__ntb) container_of(__ntb, struct amd_ntb_dev, ntb)
215#define hb_ndev(__work) container_of(__work, struct amd_ntb_dev, hb_timer.work)
216
217#endif
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c
index a198f8298258..40d04ef5da9e 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -875,7 +875,7 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
875 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar); 875 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
876 876
877 if (bar < 4 || !ndev->bar4_split) { 877 if (bar < 4 || !ndev->bar4_split) {
878 base = ioread64(mmio + base_reg); 878 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
879 879
880 /* Set the limit if supported, if size is not mw_size */ 880 /* Set the limit if supported, if size is not mw_size */
881 if (limit_reg && size != mw_size) 881 if (limit_reg && size != mw_size)
@@ -906,7 +906,7 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
906 if ((addr + size) & (~0ull << 32)) 906 if ((addr + size) & (~0ull << 32))
907 return -EINVAL; 907 return -EINVAL;
908 908
909 base = ioread32(mmio + base_reg); 909 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
910 910
911 /* Set the limit if supported, if size is not mw_size */ 911 /* Set the limit if supported, if size is not mw_size */
912 if (limit_reg && size != mw_size) 912 if (limit_reg && size != mw_size)
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h
index 2eb4addd10d0..3ec149cf6562 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.h
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.h
@@ -245,6 +245,9 @@
245#define NTB_UNSAFE_DB BIT_ULL(0) 245#define NTB_UNSAFE_DB BIT_ULL(0)
246#define NTB_UNSAFE_SPAD BIT_ULL(1) 246#define NTB_UNSAFE_SPAD BIT_ULL(1)
247 247
248#define NTB_BAR_MASK_64 ~(0xfull)
249#define NTB_BAR_MASK_32 ~(0xfu)
250
248struct intel_ntb_dev; 251struct intel_ntb_dev;
249 252
250struct intel_ntb_reg { 253struct intel_ntb_reg {
@@ -334,7 +337,8 @@ struct intel_ntb_dev {
334#define ndev_pdev(ndev) ((ndev)->ntb.pdev) 337#define ndev_pdev(ndev) ((ndev)->ntb.pdev)
335#define ndev_name(ndev) pci_name(ndev_pdev(ndev)) 338#define ndev_name(ndev) pci_name(ndev_pdev(ndev))
336#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev) 339#define ndev_dev(ndev) (&ndev_pdev(ndev)->dev)
337#define ntb_ndev(ntb) container_of(ntb, struct intel_ntb_dev, ntb) 340#define ntb_ndev(__ntb) container_of(__ntb, struct intel_ntb_dev, ntb)
338#define hb_ndev(work) container_of(work, struct intel_ntb_dev, hb_timer.work) 341#define hb_ndev(__work) container_of(__work, struct intel_ntb_dev, \
342 hb_timer.work)
339 343
340#endif 344#endif
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 60654d524858..ec4775f0ec16 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -171,12 +171,14 @@ struct ntb_transport_qp {
171 u64 rx_err_ver; 171 u64 rx_err_ver;
172 u64 rx_memcpy; 172 u64 rx_memcpy;
173 u64 rx_async; 173 u64 rx_async;
174 u64 dma_rx_prep_err;
174 u64 tx_bytes; 175 u64 tx_bytes;
175 u64 tx_pkts; 176 u64 tx_pkts;
176 u64 tx_ring_full; 177 u64 tx_ring_full;
177 u64 tx_err_no_buf; 178 u64 tx_err_no_buf;
178 u64 tx_memcpy; 179 u64 tx_memcpy;
179 u64 tx_async; 180 u64 tx_async;
181 u64 dma_tx_prep_err;
180}; 182};
181 183
182struct ntb_transport_mw { 184struct ntb_transport_mw {
@@ -249,6 +251,8 @@ enum {
249#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) 251#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
250#define NTB_QP_DEF_NUM_ENTRIES 100 252#define NTB_QP_DEF_NUM_ENTRIES 100
251#define NTB_LINK_DOWN_TIMEOUT 10 253#define NTB_LINK_DOWN_TIMEOUT 10
254#define DMA_RETRIES 20
255#define DMA_OUT_RESOURCE_TO 50
252 256
253static void ntb_transport_rxc_db(unsigned long data); 257static void ntb_transport_rxc_db(unsigned long data);
254static const struct ntb_ctx_ops ntb_transport_ops; 258static const struct ntb_ctx_ops ntb_transport_ops;
@@ -501,6 +505,12 @@ static ssize_t debugfs_read(struct file *filp, char __user *ubuf, size_t count,
501 out_offset += snprintf(buf + out_offset, out_count - out_offset, 505 out_offset += snprintf(buf + out_offset, out_count - out_offset,
502 "free tx - \t%u\n", 506 "free tx - \t%u\n",
503 ntb_transport_tx_free_entry(qp)); 507 ntb_transport_tx_free_entry(qp));
508 out_offset += snprintf(buf + out_offset, out_count - out_offset,
509 "DMA tx prep err - \t%llu\n",
510 qp->dma_tx_prep_err);
511 out_offset += snprintf(buf + out_offset, out_count - out_offset,
512 "DMA rx prep err - \t%llu\n",
513 qp->dma_rx_prep_err);
504 514
505 out_offset += snprintf(buf + out_offset, out_count - out_offset, 515 out_offset += snprintf(buf + out_offset, out_count - out_offset,
506 "\n"); 516 "\n");
@@ -726,6 +736,8 @@ static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp)
726 qp->tx_err_no_buf = 0; 736 qp->tx_err_no_buf = 0;
727 qp->tx_memcpy = 0; 737 qp->tx_memcpy = 0;
728 qp->tx_async = 0; 738 qp->tx_async = 0;
739 qp->dma_tx_prep_err = 0;
740 qp->dma_rx_prep_err = 0;
729} 741}
730 742
731static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp) 743static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp)
@@ -1228,6 +1240,7 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
1228 struct dmaengine_unmap_data *unmap; 1240 struct dmaengine_unmap_data *unmap;
1229 dma_cookie_t cookie; 1241 dma_cookie_t cookie;
1230 void *buf = entry->buf; 1242 void *buf = entry->buf;
1243 int retries = 0;
1231 1244
1232 len = entry->len; 1245 len = entry->len;
1233 1246
@@ -1263,11 +1276,21 @@ static void ntb_async_rx(struct ntb_queue_entry *entry, void *offset)
1263 1276
1264 unmap->from_cnt = 1; 1277 unmap->from_cnt = 1;
1265 1278
1266 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1], 1279 for (retries = 0; retries < DMA_RETRIES; retries++) {
1267 unmap->addr[0], len, 1280 txd = device->device_prep_dma_memcpy(chan, unmap->addr[1],
1268 DMA_PREP_INTERRUPT); 1281 unmap->addr[0], len,
1269 if (!txd) 1282 DMA_PREP_INTERRUPT);
1283 if (txd)
1284 break;
1285
1286 set_current_state(TASK_INTERRUPTIBLE);
1287 schedule_timeout(DMA_OUT_RESOURCE_TO);
1288 }
1289
1290 if (!txd) {
1291 qp->dma_rx_prep_err++;
1270 goto err_get_unmap; 1292 goto err_get_unmap;
1293 }
1271 1294
1272 txd->callback = ntb_rx_copy_callback; 1295 txd->callback = ntb_rx_copy_callback;
1273 txd->callback_param = entry; 1296 txd->callback_param = entry;
@@ -1460,6 +1483,7 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
1460 void __iomem *offset; 1483 void __iomem *offset;
1461 size_t len = entry->len; 1484 size_t len = entry->len;
1462 void *buf = entry->buf; 1485 void *buf = entry->buf;
1486 int retries = 0;
1463 1487
1464 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; 1488 offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index;
1465 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header); 1489 hdr = offset + qp->tx_max_frame - sizeof(struct ntb_payload_header);
@@ -1494,10 +1518,20 @@ static void ntb_async_tx(struct ntb_transport_qp *qp,
1494 1518
1495 unmap->to_cnt = 1; 1519 unmap->to_cnt = 1;
1496 1520
1497 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0], len, 1521 for (retries = 0; retries < DMA_RETRIES; retries++) {
1498 DMA_PREP_INTERRUPT); 1522 txd = device->device_prep_dma_memcpy(chan, dest, unmap->addr[0],
1499 if (!txd) 1523 len, DMA_PREP_INTERRUPT);
1524 if (txd)
1525 break;
1526
1527 set_current_state(TASK_INTERRUPTIBLE);
1528 schedule_timeout(DMA_OUT_RESOURCE_TO);
1529 }
1530
1531 if (!txd) {
1532 qp->dma_tx_prep_err++;
1500 goto err_get_unmap; 1533 goto err_get_unmap;
1534 }
1501 1535
1502 txd->callback = ntb_tx_copy_callback; 1536 txd->callback = ntb_tx_copy_callback;
1503 txd->callback_param = entry; 1537 txd->callback_param = entry;
@@ -1532,7 +1566,7 @@ static int ntb_process_tx(struct ntb_transport_qp *qp,
1532 1566
1533 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { 1567 if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) {
1534 if (qp->tx_handler) 1568 if (qp->tx_handler)
1535 qp->tx_handler(qp->cb_data, qp, NULL, -EIO); 1569 qp->tx_handler(qp, qp->cb_data, NULL, -EIO);
1536 1570
1537 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, 1571 ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry,
1538 &qp->tx_free_q); 1572 &qp->tx_free_q);
diff --git a/drivers/ntb/test/Kconfig b/drivers/ntb/test/Kconfig
index 01852f98a843..a5d0eda44438 100644
--- a/drivers/ntb/test/Kconfig
+++ b/drivers/ntb/test/Kconfig
@@ -17,3 +17,11 @@ config NTB_TOOL
17 functioning at a basic level. 17 functioning at a basic level.
18 18
19 If unsure, say N. 19 If unsure, say N.
20
21config NTB_PERF
22 tristate "NTB RAW Perf Measuring Tool"
23 help
24 This is a tool to measure raw NTB performance by transferring data
25 to and from the window without additional software interaction.
26
27 If unsure, say N.
diff --git a/drivers/ntb/test/Makefile b/drivers/ntb/test/Makefile
index 0ea32a324b6c..9e77e0b761c2 100644
--- a/drivers/ntb/test/Makefile
+++ b/drivers/ntb/test/Makefile
@@ -1,2 +1,3 @@
1obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o 1obj-$(CONFIG_NTB_PINGPONG) += ntb_pingpong.o
2obj-$(CONFIG_NTB_TOOL) += ntb_tool.o 2obj-$(CONFIG_NTB_TOOL) += ntb_tool.o
3obj-$(CONFIG_NTB_PERF) += ntb_perf.o
diff --git a/drivers/ntb/test/ntb_perf.c b/drivers/ntb/test/ntb_perf.c
new file mode 100644
index 000000000000..c8a37ba4b4f9
--- /dev/null
+++ b/drivers/ntb/test/ntb_perf.c
@@ -0,0 +1,748 @@
1/*
2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
4 *
5 * GPL LICENSE SUMMARY
6 *
7 * Copyright(c) 2015 Intel Corporation. All rights reserved.
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * BSD LICENSE
14 *
15 * Copyright(c) 2015 Intel Corporation. All rights reserved.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 *
21 * * Redistributions of source code must retain the above copyright
22 * notice, this list of conditions and the following disclaimer.
23 * * Redistributions in binary form must reproduce the above copy
24 * notice, this list of conditions and the following disclaimer in
25 * the documentation and/or other materials provided with the
26 * distribution.
27 * * Neither the name of Intel Corporation nor the names of its
28 * contributors may be used to endorse or promote products derived
29 * from this software without specific prior written permission.
30 *
31 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
32 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
33 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
34 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
35 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
37 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
41 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * PCIe NTB Perf Linux driver
44 */
45
46#include <linux/init.h>
47#include <linux/kernel.h>
48#include <linux/module.h>
49#include <linux/kthread.h>
50#include <linux/time.h>
51#include <linux/timer.h>
52#include <linux/dma-mapping.h>
53#include <linux/pci.h>
54#include <linux/slab.h>
55#include <linux/spinlock.h>
56#include <linux/debugfs.h>
57#include <linux/dmaengine.h>
58#include <linux/delay.h>
59#include <linux/sizes.h>
60#include <linux/ntb.h>
61
62#define DRIVER_NAME "ntb_perf"
63#define DRIVER_DESCRIPTION "PCIe NTB Performance Measurement Tool"
64
65#define DRIVER_LICENSE "Dual BSD/GPL"
66#define DRIVER_VERSION "1.0"
67#define DRIVER_AUTHOR "Dave Jiang <dave.jiang@intel.com>"
68
69#define PERF_LINK_DOWN_TIMEOUT 10
70#define PERF_VERSION 0xffff0001
71#define MAX_THREADS 32
72#define MAX_TEST_SIZE SZ_1M
73#define MAX_SRCS 32
74#define DMA_OUT_RESOURCE_TO 50
75#define DMA_RETRIES 20
76#define SZ_4G (1ULL << 32)
77#define MAX_SEG_ORDER 20 /* no larger than 1M for kmalloc buffer */
78
79MODULE_LICENSE(DRIVER_LICENSE);
80MODULE_VERSION(DRIVER_VERSION);
81MODULE_AUTHOR(DRIVER_AUTHOR);
82MODULE_DESCRIPTION(DRIVER_DESCRIPTION);
83
84static struct dentry *perf_debugfs_dir;
85
86static unsigned int seg_order = 19; /* 512K */
87module_param(seg_order, uint, 0644);
88MODULE_PARM_DESC(seg_order, "size order [n^2] of buffer segment for testing");
89
90static unsigned int run_order = 32; /* 4G */
91module_param(run_order, uint, 0644);
92MODULE_PARM_DESC(run_order, "size order [n^2] of total data to transfer");
93
94static bool use_dma; /* default to 0 */
95module_param(use_dma, bool, 0644);
96MODULE_PARM_DESC(use_dma, "Using DMA engine to measure performance");
97
98struct perf_mw {
99 phys_addr_t phys_addr;
100 resource_size_t phys_size;
101 resource_size_t xlat_align;
102 resource_size_t xlat_align_size;
103 void __iomem *vbase;
104 size_t xlat_size;
105 size_t buf_size;
106 void *virt_addr;
107 dma_addr_t dma_addr;
108};
109
110struct perf_ctx;
111
112struct pthr_ctx {
113 struct task_struct *thread;
114 struct perf_ctx *perf;
115 atomic_t dma_sync;
116 struct dma_chan *dma_chan;
117 int dma_prep_err;
118 int src_idx;
119 void *srcs[MAX_SRCS];
120};
121
122struct perf_ctx {
123 struct ntb_dev *ntb;
124 spinlock_t db_lock;
125 struct perf_mw mw;
126 bool link_is_up;
127 struct work_struct link_cleanup;
128 struct delayed_work link_work;
129 struct dentry *debugfs_node_dir;
130 struct dentry *debugfs_run;
131 struct dentry *debugfs_threads;
132 u8 perf_threads;
133 bool run;
134 struct pthr_ctx pthr_ctx[MAX_THREADS];
135 atomic_t tsync;
136};
137
138enum {
139 VERSION = 0,
140 MW_SZ_HIGH,
141 MW_SZ_LOW,
142 SPAD_MSG,
143 SPAD_ACK,
144 MAX_SPAD
145};
146
147static void perf_link_event(void *ctx)
148{
149 struct perf_ctx *perf = ctx;
150
151 if (ntb_link_is_up(perf->ntb, NULL, NULL) == 1)
152 schedule_delayed_work(&perf->link_work, 2*HZ);
153 else
154 schedule_work(&perf->link_cleanup);
155}
156
157static void perf_db_event(void *ctx, int vec)
158{
159 struct perf_ctx *perf = ctx;
160 u64 db_bits, db_mask;
161
162 db_mask = ntb_db_vector_mask(perf->ntb, vec);
163 db_bits = ntb_db_read(perf->ntb);
164
165 dev_dbg(&perf->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
166 vec, db_mask, db_bits);
167}
168
169static const struct ntb_ctx_ops perf_ops = {
170 .link_event = perf_link_event,
171 .db_event = perf_db_event,
172};
173
174static void perf_copy_callback(void *data)
175{
176 struct pthr_ctx *pctx = data;
177
178 atomic_dec(&pctx->dma_sync);
179}
180
181static ssize_t perf_copy(struct pthr_ctx *pctx, char *dst,
182 char *src, size_t size)
183{
184 struct perf_ctx *perf = pctx->perf;
185 struct dma_async_tx_descriptor *txd;
186 struct dma_chan *chan = pctx->dma_chan;
187 struct dma_device *device;
188 struct dmaengine_unmap_data *unmap;
189 dma_cookie_t cookie;
190 size_t src_off, dst_off;
191 struct perf_mw *mw = &perf->mw;
192 u64 vbase, dst_vaddr;
193 dma_addr_t dst_phys;
194 int retries = 0;
195
196 if (!use_dma) {
197 memcpy_toio(dst, src, size);
198 return size;
199 }
200
201 if (!chan) {
202 dev_err(&perf->ntb->dev, "DMA engine does not exist\n");
203 return -EINVAL;
204 }
205
206 device = chan->device;
207 src_off = (size_t)src & ~PAGE_MASK;
208 dst_off = (size_t)dst & ~PAGE_MASK;
209
210 if (!is_dma_copy_aligned(device, src_off, dst_off, size))
211 return -ENODEV;
212
213 vbase = (u64)(u64 *)mw->vbase;
214 dst_vaddr = (u64)(u64 *)dst;
215 dst_phys = mw->phys_addr + (dst_vaddr - vbase);
216
217 unmap = dmaengine_get_unmap_data(device->dev, 1, GFP_NOWAIT);
218 if (!unmap)
219 return -ENOMEM;
220
221 unmap->len = size;
222 unmap->addr[0] = dma_map_page(device->dev, virt_to_page(src),
223 src_off, size, DMA_TO_DEVICE);
224 if (dma_mapping_error(device->dev, unmap->addr[0]))
225 goto err_get_unmap;
226
227 unmap->to_cnt = 1;
228
229 do {
230 txd = device->device_prep_dma_memcpy(chan, dst_phys,
231 unmap->addr[0],
232 size, DMA_PREP_INTERRUPT);
233 if (!txd) {
234 set_current_state(TASK_INTERRUPTIBLE);
235 schedule_timeout(DMA_OUT_RESOURCE_TO);
236 }
237 } while (!txd && (++retries < DMA_RETRIES));
238
239 if (!txd) {
240 pctx->dma_prep_err++;
241 goto err_get_unmap;
242 }
243
244 txd->callback = perf_copy_callback;
245 txd->callback_param = pctx;
246 dma_set_unmap(txd, unmap);
247
248 cookie = dmaengine_submit(txd);
249 if (dma_submit_error(cookie))
250 goto err_set_unmap;
251
252 atomic_inc(&pctx->dma_sync);
253 dma_async_issue_pending(chan);
254
255 return size;
256
257err_set_unmap:
258 dmaengine_unmap_put(unmap);
259err_get_unmap:
260 dmaengine_unmap_put(unmap);
261 return 0;
262}
263
264static int perf_move_data(struct pthr_ctx *pctx, char *dst, char *src,
265 u64 buf_size, u64 win_size, u64 total)
266{
267 int chunks, total_chunks, i;
268 int copied_chunks = 0;
269 u64 copied = 0, result;
270 char *tmp = dst;
271 u64 perf, diff_us;
272 ktime_t kstart, kstop, kdiff;
273
274 chunks = div64_u64(win_size, buf_size);
275 total_chunks = div64_u64(total, buf_size);
276 kstart = ktime_get();
277
278 for (i = 0; i < total_chunks; i++) {
279 result = perf_copy(pctx, tmp, src, buf_size);
280 copied += result;
281 copied_chunks++;
282 if (copied_chunks == chunks) {
283 tmp = dst;
284 copied_chunks = 0;
285 } else
286 tmp += buf_size;
287
288 /* Probably should schedule every 4GB to prevent soft hang. */
289 if (((copied % SZ_4G) == 0) && !use_dma) {
290 set_current_state(TASK_INTERRUPTIBLE);
291 schedule_timeout(1);
292 }
293 }
294
295 if (use_dma) {
296 pr_info("%s: All DMA descriptors submitted\n", current->comm);
297 while (atomic_read(&pctx->dma_sync) != 0)
298 msleep(20);
299 }
300
301 kstop = ktime_get();
302 kdiff = ktime_sub(kstop, kstart);
303 diff_us = ktime_to_us(kdiff);
304
305 pr_info("%s: copied %llu bytes\n", current->comm, copied);
306
307 pr_info("%s: lasted %llu usecs\n", current->comm, diff_us);
308
309 perf = div64_u64(copied, diff_us);
310
311 pr_info("%s: MBytes/s: %llu\n", current->comm, perf);
312
313 return 0;
314}
315
316static bool perf_dma_filter_fn(struct dma_chan *chan, void *node)
317{
318 return dev_to_node(&chan->dev->device) == (int)(unsigned long)node;
319}
320
321static int ntb_perf_thread(void *data)
322{
323 struct pthr_ctx *pctx = data;
324 struct perf_ctx *perf = pctx->perf;
325 struct pci_dev *pdev = perf->ntb->pdev;
326 struct perf_mw *mw = &perf->mw;
327 char *dst;
328 u64 win_size, buf_size, total;
329 void *src;
330 int rc, node, i;
331 struct dma_chan *dma_chan = NULL;
332
333 pr_info("kthread %s starting...\n", current->comm);
334
335 node = dev_to_node(&pdev->dev);
336
337 if (use_dma && !pctx->dma_chan) {
338 dma_cap_mask_t dma_mask;
339
340 dma_cap_zero(dma_mask);
341 dma_cap_set(DMA_MEMCPY, dma_mask);
342 dma_chan = dma_request_channel(dma_mask, perf_dma_filter_fn,
343 (void *)(unsigned long)node);
344 if (!dma_chan) {
345 pr_warn("%s: cannot acquire DMA channel, quitting\n",
346 current->comm);
347 return -ENODEV;
348 }
349 pctx->dma_chan = dma_chan;
350 }
351
352 for (i = 0; i < MAX_SRCS; i++) {
353 pctx->srcs[i] = kmalloc_node(MAX_TEST_SIZE, GFP_KERNEL, node);
354 if (!pctx->srcs[i]) {
355 rc = -ENOMEM;
356 goto err;
357 }
358 }
359
360 win_size = mw->phys_size;
361 buf_size = 1ULL << seg_order;
362 total = 1ULL << run_order;
363
364 if (buf_size > MAX_TEST_SIZE)
365 buf_size = MAX_TEST_SIZE;
366
367 dst = (char *)mw->vbase;
368
369 atomic_inc(&perf->tsync);
370 while (atomic_read(&perf->tsync) != perf->perf_threads)
371 schedule();
372
373 src = pctx->srcs[pctx->src_idx];
374 pctx->src_idx = (pctx->src_idx + 1) & (MAX_SRCS - 1);
375
376 rc = perf_move_data(pctx, dst, src, buf_size, win_size, total);
377
378 atomic_dec(&perf->tsync);
379
380 if (rc < 0) {
381 pr_err("%s: failed\n", current->comm);
382 rc = -ENXIO;
383 goto err;
384 }
385
386 for (i = 0; i < MAX_SRCS; i++) {
387 kfree(pctx->srcs[i]);
388 pctx->srcs[i] = NULL;
389 }
390
391 return 0;
392
393err:
394 for (i = 0; i < MAX_SRCS; i++) {
395 kfree(pctx->srcs[i]);
396 pctx->srcs[i] = NULL;
397 }
398
399 if (dma_chan) {
400 dma_release_channel(dma_chan);
401 pctx->dma_chan = NULL;
402 }
403
404 return rc;
405}
406
407static void perf_free_mw(struct perf_ctx *perf)
408{
409 struct perf_mw *mw = &perf->mw;
410 struct pci_dev *pdev = perf->ntb->pdev;
411
412 if (!mw->virt_addr)
413 return;
414
415 ntb_mw_clear_trans(perf->ntb, 0);
416 dma_free_coherent(&pdev->dev, mw->buf_size,
417 mw->virt_addr, mw->dma_addr);
418 mw->xlat_size = 0;
419 mw->buf_size = 0;
420 mw->virt_addr = NULL;
421}
422
423static int perf_set_mw(struct perf_ctx *perf, resource_size_t size)
424{
425 struct perf_mw *mw = &perf->mw;
426 size_t xlat_size, buf_size;
427
428 if (!size)
429 return -EINVAL;
430
431 xlat_size = round_up(size, mw->xlat_align_size);
432 buf_size = round_up(size, mw->xlat_align);
433
434 if (mw->xlat_size == xlat_size)
435 return 0;
436
437 if (mw->buf_size)
438 perf_free_mw(perf);
439
440 mw->xlat_size = xlat_size;
441 mw->buf_size = buf_size;
442
443 mw->virt_addr = dma_alloc_coherent(&perf->ntb->pdev->dev, buf_size,
444 &mw->dma_addr, GFP_KERNEL);
445 if (!mw->virt_addr) {
446 mw->xlat_size = 0;
447 mw->buf_size = 0;
448 }
449
450 return 0;
451}
452
453static void perf_link_work(struct work_struct *work)
454{
455 struct perf_ctx *perf =
456 container_of(work, struct perf_ctx, link_work.work);
457 struct ntb_dev *ndev = perf->ntb;
458 struct pci_dev *pdev = ndev->pdev;
459 u32 val;
460 u64 size;
461 int rc;
462
463 dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);
464
465 size = perf->mw.phys_size;
466 ntb_peer_spad_write(ndev, MW_SZ_HIGH, upper_32_bits(size));
467 ntb_peer_spad_write(ndev, MW_SZ_LOW, lower_32_bits(size));
468 ntb_peer_spad_write(ndev, VERSION, PERF_VERSION);
469
470 /* now read what peer wrote */
471 val = ntb_spad_read(ndev, VERSION);
472 if (val != PERF_VERSION) {
473 dev_dbg(&pdev->dev, "Remote version = %#x\n", val);
474 goto out;
475 }
476
477 val = ntb_spad_read(ndev, MW_SZ_HIGH);
478 size = (u64)val << 32;
479
480 val = ntb_spad_read(ndev, MW_SZ_LOW);
481 size |= val;
482
483 dev_dbg(&pdev->dev, "Remote MW size = %#llx\n", size);
484
485 rc = perf_set_mw(perf, size);
486 if (rc)
487 goto out1;
488
489 perf->link_is_up = true;
490
491 return;
492
493out1:
494 perf_free_mw(perf);
495
496out:
497 if (ntb_link_is_up(ndev, NULL, NULL) == 1)
498 schedule_delayed_work(&perf->link_work,
499 msecs_to_jiffies(PERF_LINK_DOWN_TIMEOUT));
500}
501
502static void perf_link_cleanup(struct work_struct *work)
503{
504 struct perf_ctx *perf = container_of(work,
505 struct perf_ctx,
506 link_cleanup);
507
508 dev_dbg(&perf->ntb->pdev->dev, "%s called\n", __func__);
509
510 if (!perf->link_is_up)
511 cancel_delayed_work_sync(&perf->link_work);
512}
513
514static int perf_setup_mw(struct ntb_dev *ntb, struct perf_ctx *perf)
515{
516 struct perf_mw *mw;
517 int rc;
518
519 mw = &perf->mw;
520
521 rc = ntb_mw_get_range(ntb, 0, &mw->phys_addr, &mw->phys_size,
522 &mw->xlat_align, &mw->xlat_align_size);
523 if (rc)
524 return rc;
525
526 perf->mw.vbase = ioremap_wc(mw->phys_addr, mw->phys_size);
527 if (!mw->vbase)
528 return -ENOMEM;
529
530 return 0;
531}
532
533static ssize_t debugfs_run_read(struct file *filp, char __user *ubuf,
534 size_t count, loff_t *offp)
535{
536 struct perf_ctx *perf = filp->private_data;
537 char *buf;
538 ssize_t ret, out_offset;
539
540 if (!perf)
541 return 0;
542
543 buf = kmalloc(64, GFP_KERNEL);
544 out_offset = snprintf(buf, 64, "%d\n", perf->run);
545 ret = simple_read_from_buffer(ubuf, count, offp, buf, out_offset);
546 kfree(buf);
547
548 return ret;
549}
550
551static ssize_t debugfs_run_write(struct file *filp, const char __user *ubuf,
552 size_t count, loff_t *offp)
553{
554 struct perf_ctx *perf = filp->private_data;
555 int node, i;
556
557 if (!perf->link_is_up)
558 return 0;
559
560 if (perf->perf_threads == 0)
561 return 0;
562
563 if (atomic_read(&perf->tsync) == 0)
564 perf->run = false;
565
566 if (perf->run) {
567 /* lets stop the threads */
568 perf->run = false;
569 for (i = 0; i < MAX_THREADS; i++) {
570 if (perf->pthr_ctx[i].thread) {
571 kthread_stop(perf->pthr_ctx[i].thread);
572 perf->pthr_ctx[i].thread = NULL;
573 } else
574 break;
575 }
576 } else {
577 perf->run = true;
578
579 if (perf->perf_threads > MAX_THREADS) {
580 perf->perf_threads = MAX_THREADS;
581 pr_info("Reset total threads to: %u\n", MAX_THREADS);
582 }
583
584 /* no greater than 1M */
585 if (seg_order > MAX_SEG_ORDER) {
586 seg_order = MAX_SEG_ORDER;
587 pr_info("Fix seg_order to %u\n", seg_order);
588 }
589
590 if (run_order < seg_order) {
591 run_order = seg_order;
592 pr_info("Fix run_order to %u\n", run_order);
593 }
594
595 node = dev_to_node(&perf->ntb->pdev->dev);
596 /* launch kernel thread */
597 for (i = 0; i < perf->perf_threads; i++) {
598 struct pthr_ctx *pctx;
599
600 pctx = &perf->pthr_ctx[i];
601 atomic_set(&pctx->dma_sync, 0);
602 pctx->perf = perf;
603 pctx->thread =
604 kthread_create_on_node(ntb_perf_thread,
605 (void *)pctx,
606 node, "ntb_perf %d", i);
607 if (pctx->thread)
608 wake_up_process(pctx->thread);
609 else {
610 perf->run = false;
611 for (i = 0; i < MAX_THREADS; i++) {
612 if (pctx->thread) {
613 kthread_stop(pctx->thread);
614 pctx->thread = NULL;
615 }
616 }
617 }
618
619 if (perf->run == false)
620 return -ENXIO;
621 }
622
623 }
624
625 return count;
626}
627
628static const struct file_operations ntb_perf_debugfs_run = {
629 .owner = THIS_MODULE,
630 .open = simple_open,
631 .read = debugfs_run_read,
632 .write = debugfs_run_write,
633};
634
635static int perf_debugfs_setup(struct perf_ctx *perf)
636{
637 struct pci_dev *pdev = perf->ntb->pdev;
638
639 if (!debugfs_initialized())
640 return -ENODEV;
641
642 if (!perf_debugfs_dir) {
643 perf_debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
644 if (!perf_debugfs_dir)
645 return -ENODEV;
646 }
647
648 perf->debugfs_node_dir = debugfs_create_dir(pci_name(pdev),
649 perf_debugfs_dir);
650 if (!perf->debugfs_node_dir)
651 return -ENODEV;
652
653 perf->debugfs_run = debugfs_create_file("run", S_IRUSR | S_IWUSR,
654 perf->debugfs_node_dir, perf,
655 &ntb_perf_debugfs_run);
656 if (!perf->debugfs_run)
657 return -ENODEV;
658
659 perf->debugfs_threads = debugfs_create_u8("threads", S_IRUSR | S_IWUSR,
660 perf->debugfs_node_dir,
661 &perf->perf_threads);
662 if (!perf->debugfs_threads)
663 return -ENODEV;
664
665 return 0;
666}
667
668static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
669{
670 struct pci_dev *pdev = ntb->pdev;
671 struct perf_ctx *perf;
672 int node;
673 int rc = 0;
674
675 node = dev_to_node(&pdev->dev);
676
677 perf = kzalloc_node(sizeof(*perf), GFP_KERNEL, node);
678 if (!perf) {
679 rc = -ENOMEM;
680 goto err_perf;
681 }
682
683 perf->ntb = ntb;
684 perf->perf_threads = 1;
685 atomic_set(&perf->tsync, 0);
686 perf->run = false;
687 spin_lock_init(&perf->db_lock);
688 perf_setup_mw(ntb, perf);
689 INIT_DELAYED_WORK(&perf->link_work, perf_link_work);
690 INIT_WORK(&perf->link_cleanup, perf_link_cleanup);
691
692 rc = ntb_set_ctx(ntb, perf, &perf_ops);
693 if (rc)
694 goto err_ctx;
695
696 perf->link_is_up = false;
697 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
698 ntb_link_event(ntb);
699
700 rc = perf_debugfs_setup(perf);
701 if (rc)
702 goto err_ctx;
703
704 return 0;
705
706err_ctx:
707 cancel_delayed_work_sync(&perf->link_work);
708 cancel_work_sync(&perf->link_cleanup);
709 kfree(perf);
710err_perf:
711 return rc;
712}
713
714static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
715{
716 struct perf_ctx *perf = ntb->ctx;
717 int i;
718
719 dev_dbg(&perf->ntb->dev, "%s called\n", __func__);
720
721 cancel_delayed_work_sync(&perf->link_work);
722 cancel_work_sync(&perf->link_cleanup);
723
724 ntb_clear_ctx(ntb);
725 ntb_link_disable(ntb);
726
727 debugfs_remove_recursive(perf_debugfs_dir);
728 perf_debugfs_dir = NULL;
729
730 if (use_dma) {
731 for (i = 0; i < MAX_THREADS; i++) {
732 struct pthr_ctx *pctx = &perf->pthr_ctx[i];
733
734 if (pctx->dma_chan)
735 dma_release_channel(pctx->dma_chan);
736 }
737 }
738
739 kfree(perf);
740}
741
742static struct ntb_client perf_client = {
743 .ops = {
744 .probe = perf_probe,
745 .remove = perf_remove,
746 },
747};
748module_ntb_client(perf_client);
generated by cgit v1.2.2 (git 2.25.0) at 2026-04-21 08:39:36 -0400