aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSergey Temerkhanov <s.temerkhanov@gmail.com>2017-03-24 18:28:37 -0400
committerBorislav Petkov <bp@suse.de>2017-03-27 05:43:56 -0400
commit41003396f932d7f027725c7acebb6a7caa41dc3e (patch)
tree85fb52f0424c586df0344c3a58704e233d13dccc
parent819f60fb7db169d851186d04e571e9bca27321e8 (diff)
EDAC, thunderx: Add Cavium ThunderX EDAC driver
Add support for Cavium ThunderX EDAC capable on-chip peripherals, namely the DRAM controller (LMC), cache coherent processor interconnect (CCPI) and level 2 cache blocks (L2C-TAD, L2C-MCI, L2C-CBC) Signed-off-by: Sergey Temerkhanov <s.temerkhanov@gmail.com> Cc: David.Daney@cavium.com Cc: Jan.Glauber@cavium.com Cc: linux-edac <linux-edac@vger.kernel.org> Link: http://lkml.kernel.org/r/20170324222837.60583-1-s.temerkhanov@gmail.com Signed-off-by: Borislav Petkov <bp@suse.de>
Diffstat
-rw-r--r--MAINTAINERS1
-rw-r--r--drivers/edac/Kconfig11
-rw-r--r--drivers/edac/Makefile1
-rw-r--r--drivers/edac/thunderx_edac.c2183
4 files changed, 2196 insertions, 0 deletions
diff --git a/MAINTAINERS b/MAINTAINERS
index 572fe4252ac4..af89e5d0912b 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -4695,6 +4695,7 @@ L: linux-edac@vger.kernel.org
4695L: linux-mips@linux-mips.org 4695L: linux-mips@linux-mips.org
4696S: Supported 4696S: Supported
4697F: drivers/edac/octeon_edac* 4697F: drivers/edac/octeon_edac*
4698F: drivers/edac/thunderx_edac*
4698 4699
4699EDAC-E752X 4700EDAC-E752X
4700M: Mark Gross <mark.gross@intel.com> 4701M: Mark Gross <mark.gross@intel.com>
diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig
index 4773f2867234..7c68e6f955c7 100644
--- a/drivers/edac/Kconfig
+++ b/drivers/edac/Kconfig
@@ -392,6 +392,17 @@ config EDAC_OCTEON_PCI
392 Support for error detection and correction on the 392 Support for error detection and correction on the
393 Cavium Octeon family of SOCs. 393 Cavium Octeon family of SOCs.
394 394
395config EDAC_THUNDERX
396 tristate "Cavium ThunderX EDAC"
397 depends on EDAC_MM_EDAC
398 depends on ARM64
399 depends on PCI
400 help
401 Support for error detection and correction on the
402 Cavium ThunderX memory controllers (LMC), Cache
403 Coherent Processor Interconnect (CCPI) and L2 cache
404 blocks (TAD, CBC, MCI).
405
395config EDAC_ALTERA 406config EDAC_ALTERA
396 bool "Altera SOCFPGA ECC" 407 bool "Altera SOCFPGA ECC"
397 depends on EDAC_MM_EDAC=y && ARCH_SOCFPGA 408 depends on EDAC_MM_EDAC=y && ARCH_SOCFPGA
diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile
index 587107e90996..52d735f29073 100644
--- a/drivers/edac/Makefile
+++ b/drivers/edac/Makefile
@@ -74,6 +74,7 @@ obj-$(CONFIG_EDAC_OCTEON_PC) += octeon_edac-pc.o
74obj-$(CONFIG_EDAC_OCTEON_L2C) += octeon_edac-l2c.o 74obj-$(CONFIG_EDAC_OCTEON_L2C) += octeon_edac-l2c.o
75obj-$(CONFIG_EDAC_OCTEON_LMC) += octeon_edac-lmc.o 75obj-$(CONFIG_EDAC_OCTEON_LMC) += octeon_edac-lmc.o
76obj-$(CONFIG_EDAC_OCTEON_PCI) += octeon_edac-pci.o 76obj-$(CONFIG_EDAC_OCTEON_PCI) += octeon_edac-pci.o
77obj-$(CONFIG_EDAC_THUNDERX) += thunderx_edac.o
77 78
78obj-$(CONFIG_EDAC_ALTERA) += altera_edac.o 79obj-$(CONFIG_EDAC_ALTERA) += altera_edac.o
79obj-$(CONFIG_EDAC_SYNOPSYS) += synopsys_edac.o 80obj-$(CONFIG_EDAC_SYNOPSYS) += synopsys_edac.o
diff --git a/drivers/edac/thunderx_edac.c b/drivers/edac/thunderx_edac.c
new file mode 100644
index 000000000000..b5fe6894df5e
--- /dev/null
+++ b/drivers/edac/thunderx_edac.c
@@ -0,0 +1,2183 @@
1/*
2 * Cavium ThunderX memory controller kernel module
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
9 *
10 */
11
12#include <linux/module.h>
13#include <linux/pci.h>
14#include <linux/edac.h>
15#include <linux/interrupt.h>
16#include <linux/string.h>
17#include <linux/stop_machine.h>
18#include <linux/delay.h>
19#include <linux/sizes.h>
20#include <linux/atomic.h>
21#include <linux/bitfield.h>
22#include <linux/circ_buf.h>
23
24#include <asm/page.h>
25
26#include "edac_module.h"
27
28#define phys_to_pfn(phys) (PFN_DOWN(phys))
29
30#define THUNDERX_NODE GENMASK(45, 44)
31
32enum {
33 ERR_CORRECTED = 1,
34 ERR_UNCORRECTED = 2,
35 ERR_UNKNOWN = 3,
36};
37
38#define MAX_SYNDROME_REGS 4
39
40struct error_syndrome {
41 u64 reg[MAX_SYNDROME_REGS];
42};
43
44struct error_descr {
45 int type;
46 u64 mask;
47 char *descr;
48};
49
50static void decode_register(char *str, size_t size,
51 const struct error_descr *descr,
52 const uint64_t reg)
53{
54 int ret = 0;
55
56 while (descr->type && descr->mask && descr->descr) {
57 if (reg & descr->mask) {
58 ret = snprintf(str, size, "\n\t%s, %s",
59 descr->type == ERR_CORRECTED ?
60 "Corrected" : "Uncorrected",
61 descr->descr);
62 str += ret;
63 size -= ret;
64 }
65 descr++;
66 }
67}
68
69static unsigned long get_bits(unsigned long data, int pos, int width)
70{
71 return (data >> pos) & ((1 << width) - 1);
72}
73
74#define L2C_CTL 0x87E080800000
75#define L2C_CTL_DISIDXALIAS BIT(0)
76
77#define PCI_DEVICE_ID_THUNDER_LMC 0xa022
78
79#define LMC_FADR 0x20
80#define LMC_FADR_FDIMM(x) ((x >> 37) & 0x1)
81#define LMC_FADR_FBUNK(x) ((x >> 36) & 0x1)
82#define LMC_FADR_FBANK(x) ((x >> 32) & 0xf)
83#define LMC_FADR_FROW(x) ((x >> 14) & 0xffff)
84#define LMC_FADR_FCOL(x) ((x >> 0) & 0x1fff)
85
86#define LMC_NXM_FADR 0x28
87#define LMC_ECC_SYND 0x38
88
89#define LMC_ECC_PARITY_TEST 0x108
90
91#define LMC_INT_W1S 0x150
92
93#define LMC_INT_ENA_W1C 0x158
94#define LMC_INT_ENA_W1S 0x160
95
96#define LMC_CONFIG 0x188
97
98#define LMC_CONFIG_BG2 BIT(62)
99#define LMC_CONFIG_RANK_ENA BIT(42)
100#define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF)
101#define LMC_CONFIG_ROW_LSB(x) (((x) >> 2) & 0x7)
102
103#define LMC_CONTROL 0x190
104#define LMC_CONTROL_XOR_BANK BIT(16)
105
106#define LMC_INT 0x1F0
107
108#define LMC_INT_DDR_ERR BIT(11)
109#define LMC_INT_DED_ERR (0xFUL << 5)
110#define LMC_INT_SEC_ERR (0xFUL << 1)
111#define LMC_INT_NXM_WR_MASK BIT(0)
112
113#define LMC_DDR_PLL_CTL 0x258
114#define LMC_DDR_PLL_CTL_DDR4 BIT(29)
115
116#define LMC_FADR_SCRAMBLED 0x330
117
118#define LMC_INT_UE (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
119 LMC_INT_NXM_WR_MASK)
120
121#define LMC_INT_CE (LMC_INT_SEC_ERR)
122
123static const struct error_descr lmc_errors[] = {
124 {
125 .type = ERR_CORRECTED,
126 .mask = LMC_INT_SEC_ERR,
127 .descr = "Single-bit ECC error",
128 },
129 {
130 .type = ERR_UNCORRECTED,
131 .mask = LMC_INT_DDR_ERR,
132 .descr = "DDR chip error",
133 },
134 {
135 .type = ERR_UNCORRECTED,
136 .mask = LMC_INT_DED_ERR,
137 .descr = "Double-bit ECC error",
138 },
139 {
140 .type = ERR_UNCORRECTED,
141 .mask = LMC_INT_NXM_WR_MASK,
142 .descr = "Non-existent memory write",
143 },
144 {0, 0, NULL},
145};
146
147#define LMC_INT_EN_DDR_ERROR_ALERT_ENA BIT(5)
148#define LMC_INT_EN_DLCRAM_DED_ERR BIT(4)
149#define LMC_INT_EN_DLCRAM_SEC_ERR BIT(3)
150#define LMC_INT_INTR_DED_ENA BIT(2)
151#define LMC_INT_INTR_SEC_ENA BIT(1)
152#define LMC_INT_INTR_NXM_WR_ENA BIT(0)
153
154#define LMC_INT_ENA_ALL GENMASK(5, 0)
155
156#define LMC_DDR_PLL_CTL 0x258
157#define LMC_DDR_PLL_CTL_DDR4 BIT(29)
158
159#define LMC_CONTROL 0x190
160#define LMC_CONTROL_RDIMM BIT(0)
161
162#define LMC_SCRAM_FADR 0x330
163
164#define LMC_CHAR_MASK0 0x228
165#define LMC_CHAR_MASK2 0x238
166
167#define RING_ENTRIES 8
168
169struct debugfs_entry {
170 const char *name;
171 umode_t mode;
172 const struct file_operations fops;
173};
174
175struct lmc_err_ctx {
176 u64 reg_int;
177 u64 reg_fadr;
178 u64 reg_nxm_fadr;
179 u64 reg_scram_fadr;
180 u64 reg_ecc_synd;
181};
182
183struct thunderx_lmc {
184 void __iomem *regs;
185 struct pci_dev *pdev;
186 struct msix_entry msix_ent;
187
188 atomic_t ecc_int;
189
190 u64 mask0;
191 u64 mask2;
192 u64 parity_test;
193 u64 node;
194
195 int xbits;
196 int bank_width;
197 int pbank_lsb;
198 int dimm_lsb;
199 int rank_lsb;
200 int bank_lsb;
201 int row_lsb;
202 int col_hi_lsb;
203
204 int xor_bank;
205 int l2c_alias;
206
207 struct page *mem;
208
209 struct lmc_err_ctx err_ctx[RING_ENTRIES];
210 unsigned long ring_head;
211 unsigned long ring_tail;
212};
213
214#define ring_pos(pos, size) ((pos) & (size - 1))
215
216#define DEBUGFS_STRUCT(_name, _mode, _write, _read) \
217static struct debugfs_entry debugfs_##_name = { \
218 .name = __stringify(_name), \
219 .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \
220 .fops = { \
221 .open = simple_open, \
222 .write = _write, \
223 .read = _read, \
224 .llseek = generic_file_llseek, \
225 }, \
226}
227
228#define DEBUGFS_FIELD_ATTR(_type, _field) \
229static ssize_t thunderx_##_type##_##_field##_read(struct file *file, \
230 char __user *data, \
231 size_t count, loff_t *ppos) \
232{ \
233 struct thunderx_##_type *pdata = file->private_data; \
234 char buf[20]; \
235 \
236 snprintf(buf, count, "0x%016llx", pdata->_field); \
237 return simple_read_from_buffer(data, count, ppos, \
238 buf, sizeof(buf)); \
239} \
240 \
241static ssize_t thunderx_##_type##_##_field##_write(struct file *file, \
242 const char __user *data, \
243 size_t count, loff_t *ppos) \
244{ \
245 struct thunderx_##_type *pdata = file->private_data; \
246 int res; \
247 \
248 res = kstrtoull_from_user(data, count, 0, &pdata->_field); \
249 \
250 return res ? res : count; \
251} \
252 \
253DEBUGFS_STRUCT(_field, 0600, \
254 thunderx_##_type##_##_field##_write, \
255 thunderx_##_type##_##_field##_read) \
256
257#define DEBUGFS_REG_ATTR(_type, _name, _reg) \
258static ssize_t thunderx_##_type##_##_name##_read(struct file *file, \
259 char __user *data, \
260 size_t count, loff_t *ppos) \
261{ \
262 struct thunderx_##_type *pdata = file->private_data; \
263 char buf[20]; \
264 \
265 sprintf(buf, "0x%016llx", readq(pdata->regs + _reg)); \
266 return simple_read_from_buffer(data, count, ppos, \
267 buf, sizeof(buf)); \
268} \
269 \
270static ssize_t thunderx_##_type##_##_name##_write(struct file *file, \
271 const char __user *data, \
272 size_t count, loff_t *ppos) \
273{ \
274 struct thunderx_##_type *pdata = file->private_data; \
275 u64 val; \
276 int res; \
277 \
278 res = kstrtoull_from_user(data, count, 0, &val); \
279 \
280 if (!res) { \
281 writeq(val, pdata->regs + _reg); \
282 res = count; \
283 } \
284 \
285 return res; \
286} \
287 \
288DEBUGFS_STRUCT(_name, 0600, \
289 thunderx_##_type##_##_name##_write, \
290 thunderx_##_type##_##_name##_read)
291
292#define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field)
293
294/*
295 * To get an ECC error injected, the following steps are needed:
296 * - Setup the ECC injection by writing the appropriate parameters:
297 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
298 * echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
299 * echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
300 * - Do the actual injection:
301 * echo 1 > /sys/kernel/debug/<device number>/inject_ecc
302 */
303static ssize_t thunderx_lmc_inject_int_write(struct file *file,
304 const char __user *data,
305 size_t count, loff_t *ppos)
306{
307 struct thunderx_lmc *lmc = file->private_data;
308 u64 val;
309 int res;
310
311 res = kstrtoull_from_user(data, count, 0, &val);
312
313 if (!res) {
314 /* Trigger the interrupt */
315 writeq(val, lmc->regs + LMC_INT_W1S);
316 res = count;
317 }
318
319 return res;
320}
321
322static ssize_t thunderx_lmc_int_read(struct file *file,
323 char __user *data,
324 size_t count, loff_t *ppos)
325{
326 struct thunderx_lmc *lmc = file->private_data;
327 char buf[20];
328 u64 lmc_int = readq(lmc->regs + LMC_INT);
329
330 snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
331 return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
332}
333
334#define TEST_PATTERN 0xa5
335
336static int inject_ecc_fn(void *arg)
337{
338 struct thunderx_lmc *lmc = arg;
339 uintptr_t addr, phys;
340 unsigned int cline_size = cache_line_size();
341 const unsigned int lines = PAGE_SIZE / cline_size;
342 unsigned int i, cl_idx;
343
344 addr = (uintptr_t)page_address(lmc->mem);
345 phys = (uintptr_t)page_to_phys(lmc->mem);
346
347 cl_idx = (phys & 0x7f) >> 4;
348 lmc->parity_test &= ~(7ULL << 8);
349 lmc->parity_test |= (cl_idx << 8);
350
351 writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
352 writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
353 writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
354
355 readq(lmc->regs + LMC_CHAR_MASK0);
356 readq(lmc->regs + LMC_CHAR_MASK2);
357 readq(lmc->regs + LMC_ECC_PARITY_TEST);
358
359 for (i = 0; i < lines; i++) {
360 memset((void *)addr, TEST_PATTERN, cline_size);
361 barrier();
362
363 /*
364 * Flush L1 cachelines to the PoC (L2).
365 * This will cause cacheline eviction to the L2.
366 */
367 asm volatile("dc civac, %0\n"
368 "dsb sy\n"
369 : : "r"(addr + i * cline_size));
370 }
371
372 for (i = 0; i < lines; i++) {
373 /*
374 * Flush L2 cachelines to the DRAM.
375 * This will cause cacheline eviction to the DRAM
376 * and ECC corruption according to the masks set.
377 */
378 __asm__ volatile("sys #0,c11,C1,#2, %0\n"
379 : : "r"(phys + i * cline_size));
380 }
381
382 for (i = 0; i < lines; i++) {
383 /*
384 * Invalidate L2 cachelines.
385 * The subsequent load will cause cacheline fetch
386 * from the DRAM and an error interrupt
387 */
388 __asm__ volatile("sys #0,c11,C1,#1, %0"
389 : : "r"(phys + i * cline_size));
390 }
391
392 for (i = 0; i < lines; i++) {
393 /*
394 * Invalidate L1 cachelines.
395 * The subsequent load will cause cacheline fetch
396 * from the L2 and/or DRAM
397 */
398 asm volatile("dc ivac, %0\n"
399 "dsb sy\n"
400 : : "r"(addr + i * cline_size));
401 }
402
403 return 0;
404}
405
406static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
407 const char __user *data,
408 size_t count, loff_t *ppos)
409{
410 struct thunderx_lmc *lmc = file->private_data;
411
412 unsigned int cline_size = cache_line_size();
413
414 u8 tmp[cline_size];
415 void __iomem *addr;
416 unsigned int offs, timeout = 100000;
417
418 atomic_set(&lmc->ecc_int, 0);
419
420 lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
421
422 if (!lmc->mem)
423 return -ENOMEM;
424
425 addr = page_address(lmc->mem);
426
427 while (!atomic_read(&lmc->ecc_int) && timeout--) {
428 stop_machine(inject_ecc_fn, lmc, NULL);
429
430 for (offs = 0; offs < PAGE_SIZE; offs += sizeof(tmp)) {
431 /*
432 * Do a load from the previously rigged location
433 * This should generate an error interrupt.
434 */
435 memcpy(tmp, addr + offs, cline_size);
436 asm volatile("dsb ld\n");
437 }
438 }
439
440 __free_pages(lmc->mem, 0);
441
442 return count;
443}
444
445LMC_DEBUGFS_ENT(mask0);
446LMC_DEBUGFS_ENT(mask2);
447LMC_DEBUGFS_ENT(parity_test);
448
449DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
450DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
451DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
452
453struct debugfs_entry *lmc_dfs_ents[] = {
454 &debugfs_mask0,
455 &debugfs_mask2,
456 &debugfs_parity_test,
457 &debugfs_inject_ecc,
458 &debugfs_inject_int,
459 &debugfs_int_w1c,
460};
461
462static int thunderx_create_debugfs_nodes(struct dentry *parent,
463 struct debugfs_entry *attrs[],
464 void *data,
465 size_t num)
466{
467 int i;
468 struct dentry *ent;
469
470 if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
471 return 0;
472
473 if (!parent)
474 return -ENOENT;
475
476 for (i = 0; i < num; i++) {
477 ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
478 parent, data, &attrs[i]->fops);
479
480 if (!ent)
481 break;
482 }
483
484 return i;
485}
486
487static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
488{
489 phys_addr_t addr = 0;
490 int bank, xbits;
491
492 addr |= lmc->node << 40;
493 addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
494 addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
495 addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
496 addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
497
498 bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
499
500 if (lmc->xor_bank)
501 bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
502
503 addr |= bank << lmc->bank_lsb;
504
505 xbits = PCI_FUNC(lmc->pdev->devfn);
506
507 if (lmc->l2c_alias)
508 xbits ^= get_bits(addr, 20, lmc->xbits) ^
509 get_bits(addr, 12, lmc->xbits);
510
511 addr |= xbits << 7;
512
513 return addr;
514}
515
516static unsigned int thunderx_get_num_lmcs(unsigned int node)
517{
518 unsigned int number = 0;
519 struct pci_dev *pdev = NULL;
520
521 do {
522 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
523 PCI_DEVICE_ID_THUNDER_LMC,
524 pdev);
525 if (pdev) {
526#ifdef CONFIG_NUMA
527 if (pdev->dev.numa_node == node)
528 number++;
529#else
530 number++;
531#endif
532 }
533 } while (pdev);
534
535 return number;
536}
537
538#define LMC_MESSAGE_SIZE 120
539#define LMC_OTHER_SIZE (50 * ARRAY_SIZE(lmc_errors))
540
541static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
542{
543 struct mem_ctl_info *mci = dev_id;
544 struct thunderx_lmc *lmc = mci->pvt_info;
545
546 unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
547 struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
548
549 writeq(0, lmc->regs + LMC_CHAR_MASK0);
550 writeq(0, lmc->regs + LMC_CHAR_MASK2);
551 writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
552
553 ctx->reg_int = readq(lmc->regs + LMC_INT);
554 ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
555 ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
556 ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
557 ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
558
559 lmc->ring_head++;
560
561 atomic_set(&lmc->ecc_int, 1);
562
563 /* Clear the interrupt */
564 writeq(ctx->reg_int, lmc->regs + LMC_INT);
565
566 return IRQ_WAKE_THREAD;
567}
568
569static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
570{
571 struct mem_ctl_info *mci = dev_id;
572 struct thunderx_lmc *lmc = mci->pvt_info;
573 phys_addr_t phys_addr;
574
575 unsigned long tail;
576 struct lmc_err_ctx *ctx;
577
578 irqreturn_t ret = IRQ_NONE;
579
580 char *msg;
581 char *other;
582
583 msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
584 other = kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
585
586 if (!msg || !other)
587 goto err_free;
588
589 while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
590 ARRAY_SIZE(lmc->err_ctx))) {
591 tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
592
593 ctx = &lmc->err_ctx[tail];
594
595 dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
596 ctx->reg_int);
597 dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
598 ctx->reg_fadr);
599 dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
600 ctx->reg_nxm_fadr);
601 dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
602 ctx->reg_scram_fadr);
603 dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
604 ctx->reg_ecc_synd);
605
606 snprintf(msg, LMC_MESSAGE_SIZE,
607 "DIMM %lld rank %lld bank %lld row %lld col %lld",
608 LMC_FADR_FDIMM(ctx->reg_scram_fadr),
609 LMC_FADR_FBUNK(ctx->reg_scram_fadr),
610 LMC_FADR_FBANK(ctx->reg_scram_fadr),
611 LMC_FADR_FROW(ctx->reg_scram_fadr),
612 LMC_FADR_FCOL(ctx->reg_scram_fadr));
613
614 decode_register(other, LMC_OTHER_SIZE, lmc_errors,
615 ctx->reg_int);
616
617 phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
618
619 if (ctx->reg_int & LMC_INT_UE)
620 edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
621 phys_to_pfn(phys_addr),
622 offset_in_page(phys_addr),
623 0, -1, -1, -1, msg, other);
624 else if (ctx->reg_int & LMC_INT_CE)
625 edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
626 phys_to_pfn(phys_addr),
627 offset_in_page(phys_addr),
628 0, -1, -1, -1, msg, other);
629
630 lmc->ring_tail++;
631 }
632
633 ret = IRQ_HANDLED;
634
635err_free:
636 kfree(msg);
637 kfree(other);
638
639 return ret;
640}
641
642#ifdef CONFIG_PM
643static int thunderx_lmc_suspend(struct pci_dev *pdev, pm_message_t state)
644{
645 pci_save_state(pdev);
646 pci_disable_device(pdev);
647
648 pci_set_power_state(pdev, pci_choose_state(pdev, state));
649
650 return 0;
651}
652
653static int thunderx_lmc_resume(struct pci_dev *pdev)
654{
655 pci_set_power_state(pdev, PCI_D0);
656 pci_enable_wake(pdev, PCI_D0, 0);
657 pci_restore_state(pdev);
658
659 return 0;
660}
661#endif
662
663static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
664 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
665 { 0, },
666};
667
668static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
669{
670 int node = dev_to_node(&pdev->dev);
671 int ret = PCI_FUNC(pdev->devfn);
672
673 ret += max(node, 0) << 8;
674
675 return ret;
676}
677
678static int thunderx_lmc_probe(struct pci_dev *pdev,
679 const struct pci_device_id *id)
680{
681 struct thunderx_lmc *lmc;
682 struct edac_mc_layer layer;
683 struct mem_ctl_info *mci;
684 u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
685 int ret;
686 u64 lmc_int;
687 void *l2c_ioaddr;
688
689 layer.type = EDAC_MC_LAYER_SLOT;
690 layer.size = 2;
691 layer.is_virt_csrow = false;
692
693 ret = pcim_enable_device(pdev);
694 if (ret) {
695 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
696 return ret;
697 }
698
699 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
700 if (ret) {
701 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
702 return ret;
703 }
704
705 mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
706 sizeof(struct thunderx_lmc));
707 if (!mci)
708 return -ENOMEM;
709
710 mci->pdev = &pdev->dev;
711 lmc = mci->pvt_info;
712
713 pci_set_drvdata(pdev, mci);
714
715 lmc->regs = pcim_iomap_table(pdev)[0];
716
717 lmc_control = readq(lmc->regs + LMC_CONTROL);
718 lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
719 lmc_config = readq(lmc->regs + LMC_CONFIG);
720
721 if (lmc_control & LMC_CONTROL_RDIMM) {
722 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
723 lmc_ddr_pll_ctl) ?
724 MEM_RDDR4 : MEM_RDDR3;
725 } else {
726 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
727 lmc_ddr_pll_ctl) ?
728 MEM_DDR4 : MEM_DDR3;
729 }
730
731 mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
732 mci->edac_cap = EDAC_FLAG_SECDED;
733
734 mci->mod_name = "thunderx-lmc";
735 mci->mod_ver = "1";
736 mci->ctl_name = "thunderx-lmc";
737 mci->dev_name = dev_name(&pdev->dev);
738 mci->scrub_mode = SCRUB_NONE;
739
740 lmc->pdev = pdev;
741 lmc->msix_ent.entry = 0;
742
743 lmc->ring_head = 0;
744 lmc->ring_tail = 0;
745
746 ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
747 if (ret) {
748 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
749 goto err_free;
750 }
751
752 ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
753 thunderx_lmc_err_isr,
754 thunderx_lmc_threaded_isr, 0,
755 "[EDAC] ThunderX LMC", mci);
756 if (ret) {
757 dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
758 goto err_free;
759 }
760
761 lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
762
763 lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
764 lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
765 FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
766
767 lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
768 lmc->dimm_lsb = 28 + lmc->pbank_lsb + lmc->xbits;
769 lmc->rank_lsb = lmc->dimm_lsb;
770 lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
771 lmc->bank_lsb = 7 + lmc->xbits;
772 lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
773
774 lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
775
776 lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
777
778 l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node),
779 PAGE_SIZE);
780
781 if (!l2c_ioaddr) {
782 dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
783 goto err_free;
784 }
785
786 lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
787
788 iounmap(l2c_ioaddr);
789
790 ret = edac_mc_add_mc(mci);
791 if (ret) {
792 dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
793 goto err_free;
794 }
795
796 lmc_int = readq(lmc->regs + LMC_INT);
797 writeq(lmc_int, lmc->regs + LMC_INT);
798
799 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
800
801 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
802 ret = thunderx_create_debugfs_nodes(mci->debugfs,
803 lmc_dfs_ents,
804 lmc,
805 ARRAY_SIZE(lmc_dfs_ents));
806
807 if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
808 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
809 ret, ret >= 0 ? " created" : "");
810 }
811 }
812
813 return 0;
814
815err_free:
816 pci_set_drvdata(pdev, NULL);
817 edac_mc_free(mci);
818
819 return ret;
820}
821
822static void thunderx_lmc_remove(struct pci_dev *pdev)
823{
824 struct mem_ctl_info *mci = pci_get_drvdata(pdev);
825 struct thunderx_lmc *lmc = mci->pvt_info;
826
827 writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
828
829 edac_mc_del_mc(&pdev->dev);
830 edac_mc_free(mci);
831}
832
833MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
834
835static struct pci_driver thunderx_lmc_driver = {
836 .name = "thunderx_lmc_edac",
837 .probe = thunderx_lmc_probe,
838 .remove = thunderx_lmc_remove,
839#ifdef CONFIG_PM
840 .suspend = thunderx_lmc_suspend,
841 .resume = thunderx_lmc_resume,
842#endif
843 .id_table = thunderx_lmc_pci_tbl,
844};
845
846/*---------------------- OCX driver ---------------------------------*/
847
848#define PCI_DEVICE_ID_THUNDER_OCX 0xa013
849
850#define OCX_LINK_INTS 3
851#define OCX_INTS (OCX_LINK_INTS + 1)
852#define OCX_RX_LANES 24
853#define OCX_RX_LANE_STATS 15
854
855#define OCX_COM_INT 0x100
856#define OCX_COM_INT_W1S 0x108
857#define OCX_COM_INT_ENA_W1S 0x110
858#define OCX_COM_INT_ENA_W1C 0x118
859
860#define OCX_COM_IO_BADID BIT(54)
861#define OCX_COM_MEM_BADID BIT(53)
862#define OCX_COM_COPR_BADID BIT(52)
863#define OCX_COM_WIN_REQ_BADID BIT(51)
864#define OCX_COM_WIN_REQ_TOUT BIT(50)
865#define OCX_COM_RX_LANE GENMASK(23, 0)
866
867#define OCX_COM_INT_UE (0)
868
869#define OCX_COM_INT_CE (OCX_COM_IO_BADID | \
870 OCX_COM_MEM_BADID | \
871 OCX_COM_COPR_BADID | \
872 OCX_COM_WIN_REQ_BADID | \
873 OCX_COM_WIN_REQ_TOUT)
874
875static const struct error_descr ocx_com_errors[] = {
876 {
877 .type = ERR_CORRECTED,
878 .mask = OCX_COM_IO_BADID,
879 .descr = "Invalid IO transaction node ID",
880 },
881 {
882 .type = ERR_CORRECTED,
883 .mask = OCX_COM_MEM_BADID,
884 .descr = "Invalid memory transaction node ID",
885 },
886 {
887 .type = ERR_CORRECTED,
888 .mask = OCX_COM_COPR_BADID,
889 .descr = "Invalid coprocessor transaction node ID",
890 },
891 {
892 .type = ERR_CORRECTED,
893 .mask = OCX_COM_WIN_REQ_BADID,
894 .descr = "Invalid SLI transaction node ID",
895 },
896 {
897 .type = ERR_CORRECTED,
898 .mask = OCX_COM_WIN_REQ_TOUT,
899 .descr = "Window/core request timeout",
900 },
901 {0, 0, NULL},
902};
903
904#define OCX_COM_LINKX_INT(x) (0x120 + (x) * 8)
905#define OCX_COM_LINKX_INT_W1S(x) (0x140 + (x) * 8)
906#define OCX_COM_LINKX_INT_ENA_W1S(x) (0x160 + (x) * 8)
907#define OCX_COM_LINKX_INT_ENA_W1C(x) (0x180 + (x) * 8)
908
909#define OCX_COM_LINK_BAD_WORD BIT(13)
910#define OCX_COM_LINK_ALIGN_FAIL BIT(12)
911#define OCX_COM_LINK_ALIGN_DONE BIT(11)
912#define OCX_COM_LINK_UP BIT(10)
913#define OCX_COM_LINK_STOP BIT(9)
914#define OCX_COM_LINK_BLK_ERR BIT(8)
915#define OCX_COM_LINK_REINIT BIT(7)
916#define OCX_COM_LINK_LNK_DATA BIT(6)
917#define OCX_COM_LINK_RXFIFO_DBE BIT(5)
918#define OCX_COM_LINK_RXFIFO_SBE BIT(4)
919#define OCX_COM_LINK_TXFIFO_DBE BIT(3)
920#define OCX_COM_LINK_TXFIFO_SBE BIT(2)
921#define OCX_COM_LINK_REPLAY_DBE BIT(1)
922#define OCX_COM_LINK_REPLAY_SBE BIT(0)
923
924static const struct error_descr ocx_com_link_errors[] = {
925 {
926 .type = ERR_CORRECTED,
927 .mask = OCX_COM_LINK_REPLAY_SBE,
928 .descr = "Replay buffer single-bit error",
929 },
930 {
931 .type = ERR_CORRECTED,
932 .mask = OCX_COM_LINK_TXFIFO_SBE,
933 .descr = "TX FIFO single-bit error",
934 },
935 {
936 .type = ERR_CORRECTED,
937 .mask = OCX_COM_LINK_RXFIFO_SBE,
938 .descr = "RX FIFO single-bit error",
939 },
940 {
941 .type = ERR_CORRECTED,
942 .mask = OCX_COM_LINK_BLK_ERR,
943 .descr = "Block code error",
944 },
945 {
946 .type = ERR_CORRECTED,
947 .mask = OCX_COM_LINK_ALIGN_FAIL,
948 .descr = "Link alignment failure",
949 },
950 {
951 .type = ERR_CORRECTED,
952 .mask = OCX_COM_LINK_BAD_WORD,
953 .descr = "Bad code word",
954 },
955 {
956 .type = ERR_UNCORRECTED,
957 .mask = OCX_COM_LINK_REPLAY_DBE,
958 .descr = "Replay buffer double-bit error",
959 },
960 {
961 .type = ERR_UNCORRECTED,
962 .mask = OCX_COM_LINK_TXFIFO_DBE,
963 .descr = "TX FIFO double-bit error",
964 },
965 {
966 .type = ERR_UNCORRECTED,
967 .mask = OCX_COM_LINK_RXFIFO_DBE,
968 .descr = "RX FIFO double-bit error",
969 },
970 {
971 .type = ERR_UNCORRECTED,
972 .mask = OCX_COM_LINK_STOP,
973 .descr = "Link stopped",
974 },
975 {0, 0, NULL},
976};
977
978#define OCX_COM_LINK_INT_UE (OCX_COM_LINK_REPLAY_DBE | \
979 OCX_COM_LINK_TXFIFO_DBE | \
980 OCX_COM_LINK_RXFIFO_DBE | \
981 OCX_COM_LINK_STOP)
982
983#define OCX_COM_LINK_INT_CE (OCX_COM_LINK_REPLAY_SBE | \
984 OCX_COM_LINK_TXFIFO_SBE | \
985 OCX_COM_LINK_RXFIFO_SBE | \
986 OCX_COM_LINK_BLK_ERR | \
987 OCX_COM_LINK_ALIGN_FAIL | \
988 OCX_COM_LINK_BAD_WORD)
989
990#define OCX_LNE_INT(x) (0x8018 + (x) * 0x100)
991#define OCX_LNE_INT_EN(x) (0x8020 + (x) * 0x100)
992#define OCX_LNE_BAD_CNT(x) (0x8028 + (x) * 0x100)
993#define OCX_LNE_CFG(x) (0x8000 + (x) * 0x100)
994#define OCX_LNE_STAT(x, y) (0x8040 + (x) * 0x100 + (y) * 8)
995
996#define OCX_LNE_CFG_RX_BDRY_LOCK_DIS BIT(8)
997#define OCX_LNE_CFG_RX_STAT_WRAP_DIS BIT(2)
998#define OCX_LNE_CFG_RX_STAT_RDCLR BIT(1)
999#define OCX_LNE_CFG_RX_STAT_ENA BIT(0)
1000
1001
1002#define OCX_LANE_BAD_64B67B BIT(8)
1003#define OCX_LANE_DSKEW_FIFO_OVFL BIT(5)
1004#define OCX_LANE_SCRM_SYNC_LOSS BIT(4)
1005#define OCX_LANE_UKWN_CNTL_WORD BIT(3)
1006#define OCX_LANE_CRC32_ERR BIT(2)
1007#define OCX_LANE_BDRY_SYNC_LOSS BIT(1)
1008#define OCX_LANE_SERDES_LOCK_LOSS BIT(0)
1009
1010#define OCX_COM_LANE_INT_UE (0)
1011#define OCX_COM_LANE_INT_CE (OCX_LANE_SERDES_LOCK_LOSS | \
1012 OCX_LANE_BDRY_SYNC_LOSS | \
1013 OCX_LANE_CRC32_ERR | \
1014 OCX_LANE_UKWN_CNTL_WORD | \
1015 OCX_LANE_SCRM_SYNC_LOSS | \
1016 OCX_LANE_DSKEW_FIFO_OVFL | \
1017 OCX_LANE_BAD_64B67B)
1018
1019static const struct error_descr ocx_lane_errors[] = {
1020 {
1021 .type = ERR_CORRECTED,
1022 .mask = OCX_LANE_SERDES_LOCK_LOSS,
1023 .descr = "RX SerDes lock lost",
1024 },
1025 {
1026 .type = ERR_CORRECTED,
1027 .mask = OCX_LANE_BDRY_SYNC_LOSS,
1028 .descr = "RX word boundary lost",
1029 },
1030 {
1031 .type = ERR_CORRECTED,
1032 .mask = OCX_LANE_CRC32_ERR,
1033 .descr = "CRC32 error",
1034 },
1035 {
1036 .type = ERR_CORRECTED,
1037 .mask = OCX_LANE_UKWN_CNTL_WORD,
1038 .descr = "Unknown control word",
1039 },
1040 {
1041 .type = ERR_CORRECTED,
1042 .mask = OCX_LANE_SCRM_SYNC_LOSS,
1043 .descr = "Scrambler synchronization lost",
1044 },
1045 {
1046 .type = ERR_CORRECTED,
1047 .mask = OCX_LANE_DSKEW_FIFO_OVFL,
1048 .descr = "RX deskew FIFO overflow",
1049 },
1050 {
1051 .type = ERR_CORRECTED,
1052 .mask = OCX_LANE_BAD_64B67B,
1053 .descr = "Bad 64B/67B codeword",
1054 },
1055 {0, 0, NULL},
1056};
1057
1058#define OCX_LNE_INT_ENA_ALL (GENMASK(9, 8) | GENMASK(6, 0))
1059#define OCX_COM_INT_ENA_ALL (GENMASK(54, 50) | GENMASK(23, 0))
1060#define OCX_COM_LINKX_INT_ENA_ALL (GENMASK(13, 12) | \
1061 GENMASK(9, 7) | GENMASK(5, 0))
1062
1063#define OCX_TLKX_ECC_CTL(x) (0x10018 + (x) * 0x2000)
1064#define OCX_RLKX_ECC_CTL(x) (0x18018 + (x) * 0x2000)
1065
1066struct ocx_com_err_ctx {
1067 u64 reg_com_int;
1068 u64 reg_lane_int[OCX_RX_LANES];
1069 u64 reg_lane_stat11[OCX_RX_LANES];
1070};
1071
1072struct ocx_link_err_ctx {
1073 u64 reg_com_link_int;
1074 int link;
1075};
1076
1077struct thunderx_ocx {
1078 void __iomem *regs;
1079 int com_link;
1080 struct pci_dev *pdev;
1081 struct edac_device_ctl_info *edac_dev;
1082
1083 struct dentry *debugfs;
1084 struct msix_entry msix_ent[OCX_INTS];
1085
1086 struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1087 struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1088
1089 unsigned long com_ring_head;
1090 unsigned long com_ring_tail;
1091
1092 unsigned long link_ring_head;
1093 unsigned long link_ring_tail;
1094};
1095
1096#define OCX_MESSAGE_SIZE SZ_1K
1097#define OCX_OTHER_SIZE (50 * ARRAY_SIZE(ocx_com_link_errors))
1098
1099/* This handler is threaded */
1100static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1101{
1102 struct msix_entry *msix = irq_id;
1103 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1104 msix_ent[msix->entry]);
1105
1106 int lane;
1107 unsigned long head = ring_pos(ocx->com_ring_head,
1108 ARRAY_SIZE(ocx->com_err_ctx));
1109 struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1110
1111 ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1112
1113 for (lane = 0; lane < OCX_RX_LANES; lane++) {
1114 ctx->reg_lane_int[lane] =
1115 readq(ocx->regs + OCX_LNE_INT(lane));
1116 ctx->reg_lane_stat11[lane] =
1117 readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1118
1119 writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1120 }
1121
1122 writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1123
1124 ocx->com_ring_head++;
1125
1126 return IRQ_WAKE_THREAD;
1127}
1128
1129static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1130{
1131 struct msix_entry *msix = irq_id;
1132 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1133 msix_ent[msix->entry]);
1134
1135 irqreturn_t ret = IRQ_NONE;
1136
1137 unsigned long tail;
1138 struct ocx_com_err_ctx *ctx;
1139 int lane;
1140 char *msg;
1141 char *other;
1142
1143 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1144 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1145
1146 if (!msg || !other)
1147 goto err_free;
1148
1149 while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1150 ARRAY_SIZE(ocx->com_err_ctx))) {
1151 tail = ring_pos(ocx->com_ring_tail,
1152 ARRAY_SIZE(ocx->com_err_ctx));
1153 ctx = &ocx->com_err_ctx[tail];
1154
1155 snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1156 ocx->edac_dev->ctl_name, ctx->reg_com_int);
1157
1158 decode_register(other, OCX_OTHER_SIZE,
1159 ocx_com_errors, ctx->reg_com_int);
1160
1161 strncat(msg, other, OCX_MESSAGE_SIZE);
1162
1163 for (lane = 0; lane < OCX_RX_LANES; lane++)
1164 if (ctx->reg_com_int & BIT(lane)) {
1165 snprintf(other, OCX_OTHER_SIZE,
1166 "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1167 lane, ctx->reg_lane_int[lane],
1168 lane, ctx->reg_lane_stat11[lane]);
1169
1170 strncat(msg, other, OCX_MESSAGE_SIZE);
1171
1172 decode_register(other, OCX_OTHER_SIZE,
1173 ocx_lane_errors,
1174 ctx->reg_lane_int[lane]);
1175 strncat(msg, other, OCX_MESSAGE_SIZE);
1176 }
1177
1178 if (ctx->reg_com_int & OCX_COM_INT_UE)
1179 edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1180 else if (ctx->reg_com_int & OCX_COM_INT_CE)
1181 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1182
1183 ocx->com_ring_tail++;
1184 }
1185
1186 ret = IRQ_HANDLED;
1187
1188err_free:
1189 kfree(other);
1190 kfree(msg);
1191
1192 return ret;
1193}
1194
1195static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1196{
1197 struct msix_entry *msix = irq_id;
1198 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1199 msix_ent[msix->entry]);
1200 unsigned long head = ring_pos(ocx->link_ring_head,
1201 ARRAY_SIZE(ocx->link_err_ctx));
1202 struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1203
1204 ctx->link = msix->entry;
1205 ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1206
1207 writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1208
1209 ocx->link_ring_head++;
1210
1211 return IRQ_WAKE_THREAD;
1212}
1213
1214static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1215{
1216 struct msix_entry *msix = irq_id;
1217 struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1218 msix_ent[msix->entry]);
1219 irqreturn_t ret = IRQ_NONE;
1220 unsigned long tail;
1221 struct ocx_link_err_ctx *ctx;
1222
1223 char *msg;
1224 char *other;
1225
1226 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1227 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1228
1229 if (!msg || !other)
1230 goto err_free;
1231
1232 while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1233 ARRAY_SIZE(ocx->link_err_ctx))) {
1234 tail = ring_pos(ocx->link_ring_head,
1235 ARRAY_SIZE(ocx->link_err_ctx));
1236
1237 ctx = &ocx->link_err_ctx[tail];
1238
1239 snprintf(msg, OCX_MESSAGE_SIZE,
1240 "%s: OCX_COM_LINK_INT[%d]: %016llx",
1241 ocx->edac_dev->ctl_name,
1242 ctx->link, ctx->reg_com_link_int);
1243
1244 decode_register(other, OCX_OTHER_SIZE,
1245 ocx_com_link_errors, ctx->reg_com_link_int);
1246
1247 strncat(msg, other, OCX_MESSAGE_SIZE);
1248
1249 if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1250 edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1251 else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1252 edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1253
1254 ocx->link_ring_tail++;
1255 }
1256
1257 ret = IRQ_HANDLED;
1258err_free:
1259 kfree(other);
1260 kfree(msg);
1261
1262 return ret;
1263}
1264
1265#define OCX_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(ocx, _name, _reg)
1266
1267OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1268OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1269OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1270
1271OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1272OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1273OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1274
1275OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1276OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1277OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1278
1279OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1280OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1281OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1282OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1283OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1284OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1285OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1286OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1287
1288OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1289OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1290OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1291OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1292OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1293OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1294OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1295OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1296
1297OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1298OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1299OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1300OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1301OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1302OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1303OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1304OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1305
1306OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1307
1308struct debugfs_entry *ocx_dfs_ents[] = {
1309 &debugfs_tlk0_ecc_ctl,
1310 &debugfs_tlk1_ecc_ctl,
1311 &debugfs_tlk2_ecc_ctl,
1312
1313 &debugfs_rlk0_ecc_ctl,
1314 &debugfs_rlk1_ecc_ctl,
1315 &debugfs_rlk2_ecc_ctl,
1316
1317 &debugfs_com_link0_int,
1318 &debugfs_com_link1_int,
1319 &debugfs_com_link2_int,
1320
1321 &debugfs_lne00_badcnt,
1322 &debugfs_lne01_badcnt,
1323 &debugfs_lne02_badcnt,
1324 &debugfs_lne03_badcnt,
1325 &debugfs_lne04_badcnt,
1326 &debugfs_lne05_badcnt,
1327 &debugfs_lne06_badcnt,
1328 &debugfs_lne07_badcnt,
1329 &debugfs_lne08_badcnt,
1330 &debugfs_lne09_badcnt,
1331 &debugfs_lne10_badcnt,
1332 &debugfs_lne11_badcnt,
1333 &debugfs_lne12_badcnt,
1334 &debugfs_lne13_badcnt,
1335 &debugfs_lne14_badcnt,
1336 &debugfs_lne15_badcnt,
1337 &debugfs_lne16_badcnt,
1338 &debugfs_lne17_badcnt,
1339 &debugfs_lne18_badcnt,
1340 &debugfs_lne19_badcnt,
1341 &debugfs_lne20_badcnt,
1342 &debugfs_lne21_badcnt,
1343 &debugfs_lne22_badcnt,
1344 &debugfs_lne23_badcnt,
1345
1346 &debugfs_com_int,
1347};
1348
1349static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1350 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1351 { 0, },
1352};
1353
1354static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1355{
1356 int lane, stat, cfg;
1357
1358 for (lane = 0; lane < OCX_RX_LANES; lane++) {
1359 cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1360 cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1361 cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1362 writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1363
1364 for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1365 readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1366 }
1367}
1368
1369static int thunderx_ocx_probe(struct pci_dev *pdev,
1370 const struct pci_device_id *id)
1371{
1372 struct thunderx_ocx *ocx;
1373 struct edac_device_ctl_info *edac_dev;
1374 char name[32];
1375 int idx;
1376 int i;
1377 int ret;
1378 u64 reg;
1379
1380 ret = pcim_enable_device(pdev);
1381 if (ret) {
1382 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1383 return ret;
1384 }
1385
1386 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1387 if (ret) {
1388 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1389 return ret;
1390 }
1391
1392 idx = edac_device_alloc_index();
1393 snprintf(name, sizeof(name), "OCX%d", idx);
1394 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1395 name, 1, "CCPI", 1,
1396 0, NULL, 0, idx);
1397 if (!edac_dev) {
1398 dev_err(&pdev->dev, "Cannot allocate EDAC device: %d\n", ret);
1399 return -ENOMEM;
1400 }
1401 ocx = edac_dev->pvt_info;
1402 ocx->edac_dev = edac_dev;
1403 ocx->com_ring_head = 0;
1404 ocx->com_ring_tail = 0;
1405 ocx->link_ring_head = 0;
1406 ocx->link_ring_tail = 0;
1407
1408 ocx->regs = pcim_iomap_table(pdev)[0];
1409 if (!ocx->regs) {
1410 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1411 ret = -ENODEV;
1412 goto err_free;
1413 }
1414
1415 ocx->pdev = pdev;
1416
1417 for (i = 0; i < OCX_INTS; i++) {
1418 ocx->msix_ent[i].entry = i;
1419 ocx->msix_ent[i].vector = 0;
1420 }
1421
1422 ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1423 if (ret) {
1424 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1425 goto err_free;
1426 }
1427
1428 for (i = 0; i < OCX_INTS; i++) {
1429 ret = devm_request_threaded_irq(&pdev->dev,
1430 ocx->msix_ent[i].vector,
1431 (i == 3) ?
1432 thunderx_ocx_com_isr :
1433 thunderx_ocx_lnk_isr,
1434 (i == 3) ?
1435 thunderx_ocx_com_threaded_isr :
1436 thunderx_ocx_lnk_threaded_isr,
1437 0, "[EDAC] ThunderX OCX",
1438 &ocx->msix_ent[i]);
1439 if (ret)
1440 goto err_free;
1441 }
1442
1443 edac_dev->dev = &pdev->dev;
1444 edac_dev->dev_name = dev_name(&pdev->dev);
1445 edac_dev->mod_name = "thunderx-ocx";
1446 edac_dev->ctl_name = "thunderx-ocx";
1447
1448 ret = edac_device_add_device(edac_dev);
1449 if (ret) {
1450 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1451 goto err_free;
1452 }
1453
1454 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1455 ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1456
1457 ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1458 ocx_dfs_ents,
1459 ocx,
1460 ARRAY_SIZE(ocx_dfs_ents));
1461 if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1462 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1463 ret, ret >= 0 ? " created" : "");
1464 }
1465 }
1466
1467 pci_set_drvdata(pdev, edac_dev);
1468
1469 thunderx_ocx_clearstats(ocx);
1470
1471 for (i = 0; i < OCX_RX_LANES; i++) {
1472 writeq(OCX_LNE_INT_ENA_ALL,
1473 ocx->regs + OCX_LNE_INT_EN(i));
1474
1475 reg = readq(ocx->regs + OCX_LNE_INT(i));
1476 writeq(reg, ocx->regs + OCX_LNE_INT(i));
1477
1478 }
1479
1480 for (i = 0; i < OCX_LINK_INTS; i++) {
1481 reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1482 writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1483
1484 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1485 ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1486 }
1487
1488 reg = readq(ocx->regs + OCX_COM_INT);
1489 writeq(reg, ocx->regs + OCX_COM_INT);
1490
1491 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1492
1493 return 0;
1494err_free:
1495 edac_device_free_ctl_info(edac_dev);
1496
1497 return ret;
1498}
1499
1500static void thunderx_ocx_remove(struct pci_dev *pdev)
1501{
1502 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1503 struct thunderx_ocx *ocx = edac_dev->pvt_info;
1504 int i;
1505
1506 writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1507
1508 for (i = 0; i < OCX_INTS; i++) {
1509 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1510 ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1511 }
1512
1513 edac_debugfs_remove_recursive(ocx->debugfs);
1514
1515 edac_device_del_device(&pdev->dev);
1516 edac_device_free_ctl_info(edac_dev);
1517}
1518
1519MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1520
1521static struct pci_driver thunderx_ocx_driver = {
1522 .name = "thunderx_ocx_edac",
1523 .probe = thunderx_ocx_probe,
1524 .remove = thunderx_ocx_remove,
1525 .id_table = thunderx_ocx_pci_tbl,
1526};
1527
1528/*---------------------- L2C driver ---------------------------------*/
1529
1530#define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1531#define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1532#define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1533
1534#define L2C_TAD_INT_W1C 0x40000
1535#define L2C_TAD_INT_W1S 0x40008
1536
1537#define L2C_TAD_INT_ENA_W1C 0x40020
1538#define L2C_TAD_INT_ENA_W1S 0x40028
1539
1540
1541#define L2C_TAD_INT_L2DDBE BIT(1)
1542#define L2C_TAD_INT_SBFSBE BIT(2)
1543#define L2C_TAD_INT_SBFDBE BIT(3)
1544#define L2C_TAD_INT_FBFSBE BIT(4)
1545#define L2C_TAD_INT_FBFDBE BIT(5)
1546#define L2C_TAD_INT_TAGDBE BIT(9)
1547#define L2C_TAD_INT_RDDISLMC BIT(15)
1548#define L2C_TAD_INT_WRDISLMC BIT(16)
1549#define L2C_TAD_INT_LFBTO BIT(17)
1550#define L2C_TAD_INT_GSYNCTO BIT(18)
1551#define L2C_TAD_INT_RTGSBE BIT(32)
1552#define L2C_TAD_INT_RTGDBE BIT(33)
1553#define L2C_TAD_INT_RDDISOCI BIT(34)
1554#define L2C_TAD_INT_WRDISOCI BIT(35)
1555
1556#define L2C_TAD_INT_ECC (L2C_TAD_INT_L2DDBE | \
1557 L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1558 L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1559
1560#define L2C_TAD_INT_CE (L2C_TAD_INT_SBFSBE | \
1561 L2C_TAD_INT_FBFSBE)
1562
1563#define L2C_TAD_INT_UE (L2C_TAD_INT_L2DDBE | \
1564 L2C_TAD_INT_SBFDBE | \
1565 L2C_TAD_INT_FBFDBE | \
1566 L2C_TAD_INT_TAGDBE | \
1567 L2C_TAD_INT_RTGDBE | \
1568 L2C_TAD_INT_WRDISOCI | \
1569 L2C_TAD_INT_RDDISOCI | \
1570 L2C_TAD_INT_WRDISLMC | \
1571 L2C_TAD_INT_RDDISLMC | \
1572 L2C_TAD_INT_LFBTO | \
1573 L2C_TAD_INT_GSYNCTO)
1574
1575static const struct error_descr l2_tad_errors[] = {
1576 {
1577 .type = ERR_CORRECTED,
1578 .mask = L2C_TAD_INT_SBFSBE,
1579 .descr = "SBF single-bit error",
1580 },
1581 {
1582 .type = ERR_CORRECTED,
1583 .mask = L2C_TAD_INT_FBFSBE,
1584 .descr = "FBF single-bit error",
1585 },
1586 {
1587 .type = ERR_UNCORRECTED,
1588 .mask = L2C_TAD_INT_L2DDBE,
1589 .descr = "L2D double-bit error",
1590 },
1591 {
1592 .type = ERR_UNCORRECTED,
1593 .mask = L2C_TAD_INT_SBFDBE,
1594 .descr = "SBF double-bit error",
1595 },
1596 {
1597 .type = ERR_UNCORRECTED,
1598 .mask = L2C_TAD_INT_FBFDBE,
1599 .descr = "FBF double-bit error",
1600 },
1601 {
1602 .type = ERR_UNCORRECTED,
1603 .mask = L2C_TAD_INT_TAGDBE,
1604 .descr = "TAG double-bit error",
1605 },
1606 {
1607 .type = ERR_UNCORRECTED,
1608 .mask = L2C_TAD_INT_RTGDBE,
1609 .descr = "RTG double-bit error",
1610 },
1611 {
1612 .type = ERR_UNCORRECTED,
1613 .mask = L2C_TAD_INT_WRDISOCI,
1614 .descr = "Write to a disabled CCPI",
1615 },
1616 {
1617 .type = ERR_UNCORRECTED,
1618 .mask = L2C_TAD_INT_RDDISOCI,
1619 .descr = "Read from a disabled CCPI",
1620 },
1621 {
1622 .type = ERR_UNCORRECTED,
1623 .mask = L2C_TAD_INT_WRDISLMC,
1624 .descr = "Write to a disabled LMC",
1625 },
1626 {
1627 .type = ERR_UNCORRECTED,
1628 .mask = L2C_TAD_INT_RDDISLMC,
1629 .descr = "Read from a disabled LMC",
1630 },
1631 {
1632 .type = ERR_UNCORRECTED,
1633 .mask = L2C_TAD_INT_LFBTO,
1634 .descr = "LFB entry timeout",
1635 },
1636 {
1637 .type = ERR_UNCORRECTED,
1638 .mask = L2C_TAD_INT_GSYNCTO,
1639 .descr = "Global sync CCPI timeout",
1640 },
1641 {0, 0, NULL},
1642};
1643
1644#define L2C_TAD_INT_TAG (L2C_TAD_INT_TAGDBE)
1645
1646#define L2C_TAD_INT_RTG (L2C_TAD_INT_RTGDBE)
1647
1648#define L2C_TAD_INT_NXM (0)
1649
1650#define L2C_TAD_INT_DISLMC (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1651
1652#define L2C_TAD_INT_DISOCI (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1653
1654#define L2C_TAD_INT_ENA_ALL (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1655 L2C_TAD_INT_RTG | L2C_TAD_INT_NXM | \
1656 L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1657 L2C_TAD_INT_LFBTO)
1658
1659#define L2C_TAD_TIMETWO 0x50000
1660#define L2C_TAD_TIMEOUT 0x50100
1661#define L2C_TAD_ERR 0x60000
1662#define L2C_TAD_TQD_ERR 0x60100
1663#define L2C_TAD_TTG_ERR 0x60200
1664
1665
1666#define L2C_CBC_INT_W1C 0x60000
1667
1668#define L2C_CBC_INT_RSDSBE BIT(0)
1669#define L2C_CBC_INT_RSDDBE BIT(1)
1670
1671#define L2C_CBC_INT_RSD (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1672
1673#define L2C_CBC_INT_MIBSBE BIT(4)
1674#define L2C_CBC_INT_MIBDBE BIT(5)
1675
1676#define L2C_CBC_INT_MIB (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1677
1678#define L2C_CBC_INT_IORDDISOCI BIT(6)
1679#define L2C_CBC_INT_IOWRDISOCI BIT(7)
1680
1681#define L2C_CBC_INT_IODISOCI (L2C_CBC_INT_IORDDISOCI | \
1682 L2C_CBC_INT_IOWRDISOCI)
1683
1684#define L2C_CBC_INT_CE (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1685#define L2C_CBC_INT_UE (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1686
1687
1688static const struct error_descr l2_cbc_errors[] = {
1689 {
1690 .type = ERR_CORRECTED,
1691 .mask = L2C_CBC_INT_RSDSBE,
1692 .descr = "RSD single-bit error",
1693 },
1694 {
1695 .type = ERR_CORRECTED,
1696 .mask = L2C_CBC_INT_MIBSBE,
1697 .descr = "MIB single-bit error",
1698 },
1699 {
1700 .type = ERR_UNCORRECTED,
1701 .mask = L2C_CBC_INT_RSDDBE,
1702 .descr = "RSD double-bit error",
1703 },
1704 {
1705 .type = ERR_UNCORRECTED,
1706 .mask = L2C_CBC_INT_MIBDBE,
1707 .descr = "MIB double-bit error",
1708 },
1709 {
1710 .type = ERR_UNCORRECTED,
1711 .mask = L2C_CBC_INT_IORDDISOCI,
1712 .descr = "Read from a disabled CCPI",
1713 },
1714 {
1715 .type = ERR_UNCORRECTED,
1716 .mask = L2C_CBC_INT_IOWRDISOCI,
1717 .descr = "Write to a disabled CCPI",
1718 },
1719 {0, 0, NULL},
1720};
1721
1722#define L2C_CBC_INT_W1S 0x60008
1723#define L2C_CBC_INT_ENA_W1C 0x60020
1724
1725#define L2C_CBC_INT_ENA_ALL (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1726 L2C_CBC_INT_IODISOCI)
1727
1728#define L2C_CBC_INT_ENA_W1S 0x60028
1729
1730#define L2C_CBC_IODISOCIERR 0x80008
1731#define L2C_CBC_IOCERR 0x80010
1732#define L2C_CBC_RSDERR 0x80018
1733#define L2C_CBC_MIBERR 0x80020
1734
1735
1736#define L2C_MCI_INT_W1C 0x0
1737
1738#define L2C_MCI_INT_VBFSBE BIT(0)
1739#define L2C_MCI_INT_VBFDBE BIT(1)
1740
1741static const struct error_descr l2_mci_errors[] = {
1742 {
1743 .type = ERR_CORRECTED,
1744 .mask = L2C_MCI_INT_VBFSBE,
1745 .descr = "VBF single-bit error",
1746 },
1747 {
1748 .type = ERR_UNCORRECTED,
1749 .mask = L2C_MCI_INT_VBFDBE,
1750 .descr = "VBF double-bit error",
1751 },
1752 {0, 0, NULL},
1753};
1754
1755#define L2C_MCI_INT_W1S 0x8
1756#define L2C_MCI_INT_ENA_W1C 0x20
1757
1758#define L2C_MCI_INT_ENA_ALL (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1759
1760#define L2C_MCI_INT_ENA_W1S 0x28
1761
1762#define L2C_MCI_ERR 0x10000
1763
1764#define L2C_MESSAGE_SIZE SZ_1K
1765#define L2C_OTHER_SIZE (50 * ARRAY_SIZE(l2_tad_errors))
1766
1767struct l2c_err_ctx {
1768 char *reg_ext_name;
1769 u64 reg_int;
1770 u64 reg_ext;
1771};
1772
1773struct thunderx_l2c {
1774 void __iomem *regs;
1775 struct pci_dev *pdev;
1776 struct edac_device_ctl_info *edac_dev;
1777
1778 struct dentry *debugfs;
1779
1780 int index;
1781
1782 struct msix_entry msix_ent;
1783
1784 struct l2c_err_ctx err_ctx[RING_ENTRIES];
1785 unsigned long ring_head;
1786 unsigned long ring_tail;
1787};
1788
1789static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1790{
1791 struct msix_entry *msix = irq_id;
1792 struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1793 msix_ent);
1794
1795 unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1796 struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1797
1798 ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1799
1800 if (ctx->reg_int & L2C_TAD_INT_ECC) {
1801 ctx->reg_ext_name = "TQD_ERR";
1802 ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1803 } else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1804 ctx->reg_ext_name = "TTG_ERR";
1805 ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1806 } else if (ctx->reg_int & L2C_TAD_INT_NXM) {
1807 ctx->reg_ext_name = "ERR";
1808 ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1809 } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1810 ctx->reg_ext_name = "TIMEOUT";
1811 ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1812 } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1813 ctx->reg_ext_name = "ERR";
1814 ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1815 }
1816
1817 writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1818
1819 tad->ring_head++;
1820
1821 return IRQ_WAKE_THREAD;
1822}
1823
1824static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1825{
1826 struct msix_entry *msix = irq_id;
1827 struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1828 msix_ent);
1829
1830 unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1831 struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1832
1833 ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1834
1835 if (ctx->reg_int & L2C_CBC_INT_RSD) {
1836 ctx->reg_ext_name = "RSDERR";
1837 ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1838 } else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1839 ctx->reg_ext_name = "MIBERR";
1840 ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1841 } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1842 ctx->reg_ext_name = "IODISOCIERR";
1843 ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1844 }
1845
1846 writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1847
1848 cbc->ring_head++;
1849
1850 return IRQ_WAKE_THREAD;
1851}
1852
1853static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1854{
1855 struct msix_entry *msix = irq_id;
1856 struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1857 msix_ent);
1858
1859 unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1860 struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1861
1862 ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1863 ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1864
1865 writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1866
1867 ctx->reg_ext_name = "ERR";
1868
1869 mci->ring_head++;
1870
1871 return IRQ_WAKE_THREAD;
1872}
1873
1874static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1875{
1876 struct msix_entry *msix = irq_id;
1877 struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1878 msix_ent);
1879
1880 unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1881 struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1882 irqreturn_t ret = IRQ_NONE;
1883
1884 u64 mask_ue, mask_ce;
1885 const struct error_descr *l2_errors;
1886 char *reg_int_name;
1887
1888 char *msg;
1889 char *other;
1890
1891 msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1892 other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1893
1894 if (!msg || !other)
1895 goto err_free;
1896
1897 switch (l2c->pdev->device) {
1898 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1899 reg_int_name = "L2C_TAD_INT";
1900 mask_ue = L2C_TAD_INT_UE;
1901 mask_ce = L2C_TAD_INT_CE;
1902 l2_errors = l2_tad_errors;
1903 break;
1904 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1905 reg_int_name = "L2C_CBC_INT";
1906 mask_ue = L2C_CBC_INT_UE;
1907 mask_ce = L2C_CBC_INT_CE;
1908 l2_errors = l2_cbc_errors;
1909 break;
1910 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1911 reg_int_name = "L2C_MCI_INT";
1912 mask_ue = L2C_MCI_INT_VBFDBE;
1913 mask_ce = L2C_MCI_INT_VBFSBE;
1914 l2_errors = l2_mci_errors;
1915 break;
1916 default:
1917 dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1918 l2c->pdev->device);
1919 return IRQ_NONE;
1920 }
1921
1922 while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1923 ARRAY_SIZE(l2c->err_ctx))) {
1924 snprintf(msg, L2C_MESSAGE_SIZE,
1925 "%s: %s: %016llx, %s: %016llx",
1926 l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1927 ctx->reg_ext_name, ctx->reg_ext);
1928
1929 decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1930
1931 strncat(msg, other, L2C_MESSAGE_SIZE);
1932
1933 if (ctx->reg_int & mask_ue)
1934 edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1935 else if (ctx->reg_int & mask_ce)
1936 edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1937
1938 l2c->ring_tail++;
1939 }
1940
1941 return IRQ_HANDLED;
1942
1943err_free:
1944 kfree(other);
1945 kfree(msg);
1946
1947 return ret;
1948}
1949
1950#define L2C_DEBUGFS_ATTR(_name, _reg) DEBUGFS_REG_ATTR(l2c, _name, _reg)
1951
1952L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1953
1954struct debugfs_entry *l2c_tad_dfs_ents[] = {
1955 &debugfs_tad_int,
1956};
1957
1958L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1959
1960struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1961 &debugfs_cbc_int,
1962};
1963
1964L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1965
1966struct debugfs_entry *l2c_mci_dfs_ents[] = {
1967 &debugfs_mci_int,
1968};
1969
1970static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1971 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1972 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1973 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1974 { 0, },
1975};
1976
1977static int thunderx_l2c_probe(struct pci_dev *pdev,
1978 const struct pci_device_id *id)
1979{
1980 struct thunderx_l2c *l2c;
1981 struct edac_device_ctl_info *edac_dev;
1982 struct debugfs_entry **l2c_devattr;
1983 size_t dfs_entries;
1984 irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1985 char name[32];
1986 const char *fmt;
1987 u64 reg_en_offs, reg_en_mask;
1988 int idx;
1989 int ret;
1990
1991 ret = pcim_enable_device(pdev);
1992 if (ret) {
1993 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1994 return ret;
1995 }
1996
1997 ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1998 if (ret) {
1999 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
2000 return ret;
2001 }
2002
2003 switch (pdev->device) {
2004 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2005 thunderx_l2c_isr = thunderx_l2c_tad_isr;
2006 l2c_devattr = l2c_tad_dfs_ents;
2007 dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
2008 fmt = "L2C-TAD%d";
2009 reg_en_offs = L2C_TAD_INT_ENA_W1S;
2010 reg_en_mask = L2C_TAD_INT_ENA_ALL;
2011 break;
2012 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2013 thunderx_l2c_isr = thunderx_l2c_cbc_isr;
2014 l2c_devattr = l2c_cbc_dfs_ents;
2015 dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
2016 fmt = "L2C-CBC%d";
2017 reg_en_offs = L2C_CBC_INT_ENA_W1S;
2018 reg_en_mask = L2C_CBC_INT_ENA_ALL;
2019 break;
2020 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2021 thunderx_l2c_isr = thunderx_l2c_mci_isr;
2022 l2c_devattr = l2c_mci_dfs_ents;
2023 dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
2024 fmt = "L2C-MCI%d";
2025 reg_en_offs = L2C_MCI_INT_ENA_W1S;
2026 reg_en_mask = L2C_MCI_INT_ENA_ALL;
2027 break;
2028 default:
2029 //Should never ever get here
2030 dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
2031 pdev->device);
2032 return -EINVAL;
2033 }
2034
2035 idx = edac_device_alloc_index();
2036 snprintf(name, sizeof(name), fmt, idx);
2037
2038 edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
2039 name, 1, "L2C", 1, 0,
2040 NULL, 0, idx);
2041 if (!edac_dev) {
2042 dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2043 return -ENOMEM;
2044 }
2045
2046 l2c = edac_dev->pvt_info;
2047 l2c->edac_dev = edac_dev;
2048
2049 l2c->regs = pcim_iomap_table(pdev)[0];
2050 if (!l2c->regs) {
2051 dev_err(&pdev->dev, "Cannot map PCI resources\n");
2052 ret = -ENODEV;
2053 goto err_free;
2054 }
2055
2056 l2c->pdev = pdev;
2057
2058 l2c->ring_head = 0;
2059 l2c->ring_tail = 0;
2060
2061 l2c->msix_ent.entry = 0;
2062 l2c->msix_ent.vector = 0;
2063
2064 ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2065 if (ret) {
2066 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2067 goto err_free;
2068 }
2069
2070 ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2071 thunderx_l2c_isr,
2072 thunderx_l2c_threaded_isr,
2073 0, "[EDAC] ThunderX L2C",
2074 &l2c->msix_ent);
2075 if (ret)
2076 goto err_free;
2077
2078 edac_dev->dev = &pdev->dev;
2079 edac_dev->dev_name = dev_name(&pdev->dev);
2080 edac_dev->mod_name = "thunderx-l2c";
2081 edac_dev->ctl_name = "thunderx-l2c";
2082
2083 ret = edac_device_add_device(edac_dev);
2084 if (ret) {
2085 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2086 goto err_free;
2087 }
2088
2089 if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2090 l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2091
2092 thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2093 l2c, dfs_entries);
2094
2095 if (ret != dfs_entries) {
2096 dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2097 ret, ret >= 0 ? " created" : "");
2098 }
2099 }
2100
2101 pci_set_drvdata(pdev, edac_dev);
2102
2103 writeq(reg_en_mask, l2c->regs + reg_en_offs);
2104
2105 return 0;
2106
2107err_free:
2108 edac_device_free_ctl_info(edac_dev);
2109
2110 return ret;
2111}
2112
2113static void thunderx_l2c_remove(struct pci_dev *pdev)
2114{
2115 struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2116 struct thunderx_l2c *l2c = edac_dev->pvt_info;
2117
2118 switch (pdev->device) {
2119 case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2120 writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2121 break;
2122 case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2123 writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2124 break;
2125 case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2126 writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2127 break;
2128 }
2129
2130 edac_debugfs_remove_recursive(l2c->debugfs);
2131
2132 edac_device_del_device(&pdev->dev);
2133 edac_device_free_ctl_info(edac_dev);
2134}
2135
2136MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2137
2138static struct pci_driver thunderx_l2c_driver = {
2139 .name = "thunderx_l2c_edac",
2140 .probe = thunderx_l2c_probe,
2141 .remove = thunderx_l2c_remove,
2142 .id_table = thunderx_l2c_pci_tbl,
2143};
2144
2145static int __init thunderx_edac_init(void)
2146{
2147 int rc = 0;
2148
2149 rc = pci_register_driver(&thunderx_lmc_driver);
2150 if (rc)
2151 return rc;
2152
2153 rc = pci_register_driver(&thunderx_ocx_driver);
2154 if (rc)
2155 goto err_lmc;
2156
2157 rc = pci_register_driver(&thunderx_l2c_driver);
2158 if (rc)
2159 goto err_ocx;
2160
2161 return rc;
2162err_ocx:
2163 pci_unregister_driver(&thunderx_ocx_driver);
2164err_lmc:
2165 pci_unregister_driver(&thunderx_lmc_driver);
2166
2167 return rc;
2168}
2169
2170static void __exit thunderx_edac_exit(void)
2171{
2172 pci_unregister_driver(&thunderx_l2c_driver);
2173 pci_unregister_driver(&thunderx_ocx_driver);
2174 pci_unregister_driver(&thunderx_lmc_driver);
2175
2176}
2177
2178module_init(thunderx_edac_init);
2179module_exit(thunderx_edac_exit);
2180
2181MODULE_LICENSE("GPL v2");
2182MODULE_AUTHOR("Cavium, Inc.");
2183MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-22 12:02:47 -0400