summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorThomas Fleury <tfleury@nvidia.com>2018-01-23 17:20:43 -0500
committermobile promotions <svcmobile_promotions@nvidia.com>2018-02-26 00:48:24 -0500
commit0601fd25a5e01d0da638efef13c58b64f198bafb (patch)
treedbe843912a525497103bbe9beefc6e78ff67c8c9
parent223ea4d8a179835dd5899bbc12fe78b4998b0bd7 (diff)
gpu: nvgpu: gv100: nvlink endpoint driver
The following changes implements the initial (as per bringup) nvlink driver. (1) SW initialization of nvlink core driver structures (2) Nvlink interrupt handling (3) Device initialization (IOCTRL, pll and clocks, device level intr) (4) Falcon support for minion (5) Minion load and bootstrapping (6) Link initialization and DL PROD settings (7) Device Interface init (and switching HSHUB to nvlink) (8) HS set/get mode for both link and sublink (9) Topology discovery and VBIOS settings. (10) Ensures we get physical contiguous memory when Nvlink is enabled This driver includes a hack for the current single dev/single link limitation. JIRA: EVLR-2331 JIRA: EVLR-2330 JIRA: EVLR-2329 JIRA: EVLR-2328 Change-Id: Idca9a819179376cc655784482b24b575a52fa9e5 Signed-off-by: Thomas Fleury <tfleury@nvidia.com> Reviewed-on: https://git-master.nvidia.com/r/1656790 Reviewed-by: mobile promotions <svcmobile_promotions@nvidia.com> Tested-by: mobile promotions <svcmobile_promotions@nvidia.com>
Diffstat
-rw-r--r--drivers/gpu/nvgpu/Makefile3
-rw-r--r--drivers/gpu/nvgpu/common/linux/nvgpu_mem.c4
-rw-r--r--drivers/gpu/nvgpu/common/linux/nvlink.c46
-rw-r--r--drivers/gpu/nvgpu/gk20a/gk20a.h10
-rw-r--r--drivers/gpu/nvgpu/gv100/flcn_gv100.c55
-rw-r--r--drivers/gpu/nvgpu/gv100/flcn_gv100.h27
-rw-r--r--drivers/gpu/nvgpu/gv100/hal_gv100.c49
-rw-r--r--drivers/gpu/nvgpu/gv100/mc_gv100.c94
-rw-r--r--drivers/gpu/nvgpu/gv100/mc_gv100.h30
-rw-r--r--drivers/gpu/nvgpu/gv100/nvlink_gv100.c2654
-rw-r--r--drivers/gpu/nvgpu/gv100/nvlink_gv100.h60
-rw-r--r--drivers/gpu/nvgpu/gv11b/fifo_gv11b.c2
-rw-r--r--drivers/gpu/nvgpu/gv11b/hal_gv11b.c2
-rw-r--r--drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h8
-rw-r--r--drivers/gpu/nvgpu/include/nvgpu/nvlink.h15
15 files changed, 3016 insertions, 43 deletions
diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile
index 4169bc9b..d9d1da95 100644
--- a/drivers/gpu/nvgpu/Makefile
+++ b/drivers/gpu/nvgpu/Makefile
@@ -246,6 +246,9 @@ nvgpu-y += \
246 gv100/fifo_gv100.o \ 246 gv100/fifo_gv100.o \
247 gv100/gr_gv100.o \ 247 gv100/gr_gv100.o \
248 gv100/regops_gv100.o \ 248 gv100/regops_gv100.o \
249 gv100/flcn_gv100.o \
250 gv100/mc_gv100.o \
251 gv100/nvlink_gv100.o \
249 gv100/hal_gv100.o \ 252 gv100/hal_gv100.o \
250 gv100/pmu_gv100.o \ 253 gv100/pmu_gv100.o \
251 pstate/pstate.o \ 254 pstate/pstate.o \
diff --git a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
index e4b4be06..206b83e1 100644
--- a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
+++ b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -536,6 +536,8 @@ static u64 nvgpu_mem_linux_sgl_gpu_addr(struct gk20a *g, void *sgl,
536static bool nvgpu_mem_linux_sgt_iommuable(struct gk20a *g, 536static bool nvgpu_mem_linux_sgt_iommuable(struct gk20a *g,
537 struct nvgpu_sgt *sgt) 537 struct nvgpu_sgt *sgt)
538{ 538{
539 if (nvgpu_is_enabled(g, NVGPU_MM_USE_PHYSICAL_SG))
540 return false;
539 return true; 541 return true;
540} 542}
541 543
diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c
index 6cf74048..61dc8bc7 100644
--- a/drivers/gpu/nvgpu/common/linux/nvlink.c
+++ b/drivers/gpu/nvgpu/common/linux/nvlink.c
@@ -40,7 +40,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev)
40 40
41 /* Lets find the detected link */ 41 /* Lets find the detected link */
42 if (g->nvlink.initialized_links) 42 if (g->nvlink.initialized_links)
43 link_id = fls(g->nvlink.initialized_links); 43 link_id = ffs(g->nvlink.initialized_links) - 1;
44 else 44 else
45 return NVLINK_MAX_LINKS_SW; 45 return NVLINK_MAX_LINKS_SW;
46 46
@@ -49,6 +49,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev)
49 49
50 return NVLINK_MAX_LINKS_SW; 50 return NVLINK_MAX_LINKS_SW;
51} 51}
52
52static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) 53static int nvgpu_nvlink_early_init(struct nvlink_device *ndev)
53{ 54{
54 struct gk20a *g = (struct gk20a *) ndev->priv; 55 struct gk20a *g = (struct gk20a *) ndev->priv;
@@ -75,9 +76,10 @@ static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev)
75 * First check the topology and setup connectivity 76 * First check the topology and setup connectivity
76 * HACK: we are only enabling one link for now!!! 77 * HACK: we are only enabling one link for now!!!
77 */ 78 */
78 link_id = fls(g->nvlink.discovered_links); 79 link_id = ffs(g->nvlink.discovered_links) - 1;
79 g->nvlink.links[link_id].remote_info.is_connected = true; 80 g->nvlink.links[link_id].remote_info.is_connected = true;
80 81 g->nvlink.links[link_id].remote_info.device_type =
82 nvgpu_nvlink_endp_tegra;
81 err = g->ops.nvlink.link_early_init(g, BIT(link_id)); 83 err = g->ops.nvlink.link_early_init(g, BIT(link_id));
82 84
83 if (err == 0) { 85 if (err == 0) {
@@ -262,6 +264,7 @@ static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev,
262 return -EINVAL; 264 return -EINVAL;
263 265
264 mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); 266 mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink);
267
265 switch (mode) { 268 switch (mode) {
266 case nvgpu_nvlink_sublink_tx_hs: 269 case nvgpu_nvlink_sublink_tx_hs:
267 return NVLINK_TX_HS; 270 return NVLINK_TX_HS;
@@ -386,9 +389,9 @@ static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev,
386} 389}
387#endif 390#endif
388 391
389u32 nvgpu_nvlink_enumerate(struct gk20a *g) 392int nvgpu_nvlink_enumerate(struct gk20a *g)
390{ 393{
391 u32 err = -ENODEV; 394 int err = -ENODEV;
392#ifdef CONFIG_TEGRA_NVLINK 395#ifdef CONFIG_TEGRA_NVLINK
393 struct nvlink_device *ndev; 396 struct nvlink_device *ndev;
394 397
@@ -400,9 +403,9 @@ u32 nvgpu_nvlink_enumerate(struct gk20a *g)
400 return err; 403 return err;
401} 404}
402 405
403u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) 406int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off)
404{ 407{
405 u32 err = -ENODEV; 408 int err = -ENODEV;
406#ifdef CONFIG_TEGRA_NVLINK 409#ifdef CONFIG_TEGRA_NVLINK
407 struct nvlink_device *ndev; 410 struct nvlink_device *ndev;
408 411
@@ -423,10 +426,10 @@ u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off)
423 return err; 426 return err;
424} 427}
425 428
426u32 nvgpu_nvlink_probe(struct gk20a *g) 429int nvgpu_nvlink_probe(struct gk20a *g)
427{ 430{
428#ifdef CONFIG_TEGRA_NVLINK 431#ifdef CONFIG_TEGRA_NVLINK
429 u32 err = 0; 432 int err = 0;
430 struct device_node *np = nvgpu_get_node(g); 433 struct device_node *np = nvgpu_get_node(g);
431 struct device_node *nvlink_np = NULL, *endp_np = NULL; 434 struct device_node *nvlink_np = NULL, *endp_np = NULL;
432 struct nvlink_device *ndev; 435 struct nvlink_device *ndev;
@@ -435,13 +438,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g)
435 /* Parse DT */ 438 /* Parse DT */
436 if (np) { 439 if (np) {
437 nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); 440 nvlink_np = of_get_child_by_name(np, "nvidia,nvlink");
438 if (nvlink_np) 441 if (nvlink_np) {
439 endp_np = of_get_child_by_name(np, "endpoint"); 442 endp_np = of_get_child_by_name(nvlink_np, "endpoint");
443 }
440 } 444 }
441 445
442 if (!endp_np) { 446 if (!endp_np) {
443 nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, 447 nvgpu_info(g, "no nvlink DT detected");
444 "No Nvlink DT detected");
445 return -ENODEV; 448 return -ENODEV;
446 } 449 }
447 450
@@ -466,13 +469,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g)
466 of_property_read_u32(endp_np, "physical_link", 469 of_property_read_u32(endp_np, "physical_link",
467 &phys_link_id); 470 &phys_link_id);
468 471
469 g->nvlink.topology_connected_links = BIT(phys_link_id); 472 g->nvlink.connected_links = BIT(phys_link_id);
470 473
471 /* Check that we are in dGPU mode */ 474 /* Check that we are in dGPU mode */
472 if (ndev->device_id != NVLINK_ENDPT_GV100) { 475 if (ndev->device_id != NVLINK_ENDPT_GV100) {
473 nvgpu_err(g, "Local nvlink device is not dGPU"); 476 nvgpu_err(g, "Local nvlink device is not dGPU");
474 err = -EINVAL; 477 err = -EINVAL;
475 goto free_nvlink; 478 goto free_ndev;
476 } 479 }
477 480
478 /* Fill in device struct */ 481 /* Fill in device struct */
@@ -500,22 +503,27 @@ u32 nvgpu_nvlink_probe(struct gk20a *g)
500 err = nvlink_register_device(ndev); 503 err = nvlink_register_device(ndev);
501 if (err) { 504 if (err) {
502 nvgpu_err(g, "failed on nvlink device registration"); 505 nvgpu_err(g, "failed on nvlink device registration");
503 goto free_nvlink; 506 goto free_ndev;
504 } 507 }
505 508
506 /* Register link with core driver */ 509 /* Register link with core driver */
507 err = nvlink_register_link(&ndev->link); 510 err = nvlink_register_link(&ndev->link);
508 if (err) { 511 if (err) {
509 nvgpu_err(g, "failed on nvlink link registration"); 512 nvgpu_err(g, "failed on nvlink link registration");
510 goto free_nvlink; 513 goto unregister_ndev;
511 } 514 }
512 515
513 /* Enable NVLINK support */ 516 /* Enable NVLINK support */
514 __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); 517 __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true);
515free_nvlink: 518 return 0;
519
520unregister_ndev:
521 nvlink_unregister_device(ndev);
522
523free_ndev:
516 nvgpu_kfree(g, ndev); 524 nvgpu_kfree(g, ndev);
525 g->nvlink.priv = NULL;
517 return err; 526 return err;
518
519#else 527#else
520 return -ENODEV; 528 return -ENODEV;
521#endif 529#endif
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 02c7d0d9..40656edd 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -967,6 +967,8 @@ struct gpu_ops {
967 void (*isr_stall)(struct gk20a *g); 967 void (*isr_stall)(struct gk20a *g);
968 bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); 968 bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr);
969 bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); 969 bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr);
970 bool (*is_stall_and_eng_intr_pending)(struct gk20a *g,
971 u32 act_eng_id);
970 u32 (*intr_stall)(struct gk20a *g); 972 u32 (*intr_stall)(struct gk20a *g);
971 void (*intr_stall_pause)(struct gk20a *g); 973 void (*intr_stall_pause)(struct gk20a *g);
972 void (*intr_stall_resume)(struct gk20a *g); 974 void (*intr_stall_resume)(struct gk20a *g);
@@ -1064,10 +1066,10 @@ struct gpu_ops {
1064 int (*check_priv_security)(struct gk20a *g); 1066 int (*check_priv_security)(struct gk20a *g);
1065 } fuse; 1067 } fuse;
1066 struct { 1068 struct {
1067 u32 (*init)(struct gk20a *g); 1069 int (*init)(struct gk20a *g);
1068 u32 (*discover_ioctrl)(struct gk20a *g); 1070 int (*discover_ioctrl)(struct gk20a *g);
1069 u32 (*discover_link)(struct gk20a *g); 1071 int (*discover_link)(struct gk20a *g);
1070 u32 (*isr)(struct gk20a *g); 1072 int (*isr)(struct gk20a *g);
1071 /* API */ 1073 /* API */
1072 int (*link_early_init)(struct gk20a *g, unsigned long mask); 1074 int (*link_early_init)(struct gk20a *g, unsigned long mask);
1073 u32 (*link_get_mode)(struct gk20a *g, u32 link_id); 1075 u32 (*link_get_mode)(struct gk20a *g, u32 link_id);
diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.c b/drivers/gpu/nvgpu/gv100/flcn_gv100.c
new file mode 100644
index 00000000..5820d6c9
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.c
@@ -0,0 +1,55 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22#include <nvgpu/falcon.h>
23#include <nvgpu/pmu.h>
24
25#include "gk20a/gk20a.h"
26#include "gk20a/flcn_gk20a.h"
27#include "gp106/flcn_gp106.h"
28#include "gv100/flcn_gv100.h"
29
30#include <nvgpu/hw/gv100/hw_falcon_gv100.h>
31
32void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn)
33{
34 struct gk20a *g = flcn->g;
35
36 switch (flcn->flcn_id) {
37 case FALCON_ID_MINION:
38 flcn->flcn_base = g->nvlink.minion_base;
39 flcn->is_falcon_supported = true;
40 flcn->is_interrupt_enabled = true;
41 break;
42 default:
43 break;
44 }
45
46 if (flcn->is_falcon_supported) {
47 nvgpu_mutex_init(&flcn->copy_lock);
48 gk20a_falcon_ops(flcn);
49 } else {
50 /*
51 * Fall back
52 */
53 gp106_falcon_hal_sw_init(flcn);
54 }
55}
diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.h b/drivers/gpu/nvgpu/gv100/flcn_gv100.h
new file mode 100644
index 00000000..869e74c8
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.h
@@ -0,0 +1,27 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22#ifndef __FLCN_GV100_H__
23#define __FLCN_GV100_H__
24
25void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn);
26
27#endif /* __FLCN_GV100_H__ */
diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c
index 6f4ab875..53d61bfb 100644
--- a/drivers/gpu/nvgpu/gv100/hal_gv100.c
+++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c
@@ -51,20 +51,19 @@
51#include "gm20b/pmu_gm20b.h" 51#include "gm20b/pmu_gm20b.h"
52#include "gm20b/acr_gm20b.h" 52#include "gm20b/acr_gm20b.h"
53 53
54#include "gp10b/fb_gp10b.h"
55#include "gp10b/gr_gp10b.h"
56
57#include "gp106/clk_gp106.h" 54#include "gp106/clk_gp106.h"
58#include "gp106/clk_arb_gp106.h" 55#include "gp106/clk_arb_gp106.h"
59#include "gp106/pmu_gp106.h" 56#include "gp106/pmu_gp106.h"
60#include "gp106/acr_gp106.h" 57#include "gp106/acr_gp106.h"
61#include "gp106/sec2_gp106.h" 58#include "gp106/sec2_gp106.h"
62#include "gp106/bios_gp106.h" 59#include "gp106/bios_gp106.h"
63#include "gv100/bios_gv100.h"
64#include "gp106/therm_gp106.h" 60#include "gp106/therm_gp106.h"
65#include "gp106/xve_gp106.h" 61#include "gp106/xve_gp106.h"
66#include "gp106/clk_gp106.h" 62#include "gp106/clk_gp106.h"
67#include "gp106/flcn_gp106.h" 63#include "gp106/flcn_gp106.h"
64
65#include "gp10b/fb_gp10b.h"
66#include "gp10b/gr_gp10b.h"
68#include "gp10b/ltc_gp10b.h" 67#include "gp10b/ltc_gp10b.h"
69#include "gp10b/therm_gp10b.h" 68#include "gp10b/therm_gp10b.h"
70#include "gp10b/mc_gp10b.h" 69#include "gp10b/mc_gp10b.h"
@@ -78,32 +77,33 @@
78#include "gv11b/css_gr_gv11b.h" 77#include "gv11b/css_gr_gv11b.h"
79#include "gv11b/dbg_gpu_gv11b.h" 78#include "gv11b/dbg_gpu_gv11b.h"
80#include "gv11b/hal_gv11b.h" 79#include "gv11b/hal_gv11b.h"
81#include "gv100/gr_gv100.h"
82#include "gv11b/gr_gv11b.h" 80#include "gv11b/gr_gv11b.h"
83#include "gv11b/mc_gv11b.h" 81#include "gv11b/mc_gv11b.h"
84#include "gv11b/ltc_gv11b.h" 82#include "gv11b/ltc_gv11b.h"
85#include "gv11b/gv11b.h" 83#include "gv11b/gv11b.h"
86#include "gv11b/ce_gv11b.h" 84#include "gv11b/ce_gv11b.h"
87#include "gv100/gr_ctx_gv100.h"
88#include "gv11b/mm_gv11b.h" 85#include "gv11b/mm_gv11b.h"
89#include "gv11b/pmu_gv11b.h" 86#include "gv11b/pmu_gv11b.h"
90#include "gv11b/fb_gv11b.h" 87#include "gv11b/fb_gv11b.h"
91#include "gv100/mm_gv100.h"
92#include "gv11b/pmu_gv11b.h" 88#include "gv11b/pmu_gv11b.h"
93#include "gv100/fb_gv100.h"
94#include "gv100/fifo_gv100.h"
95#include "gv11b/fifo_gv11b.h" 89#include "gv11b/fifo_gv11b.h"
96#include "gv11b/regops_gv11b.h" 90#include "gv11b/regops_gv11b.h"
97
98#include "gv11b/gv11b_gating_reglist.h" 91#include "gv11b/gv11b_gating_reglist.h"
99#include "gv100/regops_gv100.h"
100#include "gv11b/subctx_gv11b.h" 92#include "gv11b/subctx_gv11b.h"
101 93
102#include "gv100.h" 94#include "gv100.h"
103#include "hal_gv100.h" 95#include "hal_gv100.h"
96#include "gv100/bios_gv100.h"
104#include "gv100/fb_gv100.h" 97#include "gv100/fb_gv100.h"
98#include "gv100/fifo_gv100.h"
99#include "gv100/flcn_gv100.h"
100#include "gv100/gr_ctx_gv100.h"
101#include "gv100/gr_gv100.h"
102#include "gv100/mc_gv100.h"
105#include "gv100/mm_gv100.h" 103#include "gv100/mm_gv100.h"
106#include "gv100/pmu_gv100.h" 104#include "gv100/pmu_gv100.h"
105#include "gv100/nvlink_gv100.h"
106#include "gv100/regops_gv100.h"
107 107
108#include <nvgpu/bus.h> 108#include <nvgpu/bus.h>
109#include <nvgpu/debug.h> 109#include <nvgpu/debug.h>
@@ -651,7 +651,7 @@ static const struct gpu_ops gv100_ops = {
651 .apply_smpc_war = gv100_apply_smpc_war, 651 .apply_smpc_war = gv100_apply_smpc_war,
652 }, 652 },
653 .mc = { 653 .mc = {
654 .intr_enable = mc_gv11b_intr_enable, 654 .intr_enable = mc_gv100_intr_enable,
655 .intr_unit_config = mc_gp10b_intr_unit_config, 655 .intr_unit_config = mc_gp10b_intr_unit_config,
656 .isr_stall = mc_gp10b_isr_stall, 656 .isr_stall = mc_gp10b_isr_stall,
657 .intr_stall = mc_gp10b_intr_stall, 657 .intr_stall = mc_gp10b_intr_stall,
@@ -666,6 +666,9 @@ static const struct gpu_ops gv100_ops = {
666 .boot_0 = gk20a_mc_boot_0, 666 .boot_0 = gk20a_mc_boot_0,
667 .is_intr1_pending = mc_gp10b_is_intr1_pending, 667 .is_intr1_pending = mc_gp10b_is_intr1_pending,
668 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, 668 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending,
669 .is_intr_nvlink_pending = gv100_mc_is_intr_nvlink_pending,
670 .is_stall_and_eng_intr_pending =
671 gv100_mc_is_stall_and_eng_intr_pending,
669 }, 672 },
670 .debug = { 673 .debug = {
671 .show_dump = gk20a_debug_show_dump, 674 .show_dump = gk20a_debug_show_dump,
@@ -712,11 +715,30 @@ static const struct gpu_ops gv100_ops = {
712 .disable_shadow_rom = xve_disable_shadow_rom_gp106, 715 .disable_shadow_rom = xve_disable_shadow_rom_gp106,
713 }, 716 },
714 .falcon = { 717 .falcon = {
715 .falcon_hal_sw_init = gp106_falcon_hal_sw_init, 718 .falcon_hal_sw_init = gv100_falcon_hal_sw_init,
716 }, 719 },
717 .priv_ring = { 720 .priv_ring = {
718 .isr = gp10b_priv_ring_isr, 721 .isr = gp10b_priv_ring_isr,
719 }, 722 },
723 .nvlink = {
724 .discover_ioctrl = gv100_nvlink_discover_ioctrl,
725 .discover_link = gv100_nvlink_discover_link,
726 .init = gv100_nvlink_init,
727 .isr = gv100_nvlink_isr,
728 /* API */
729 .link_early_init = gv100_nvlink_link_early_init,
730 .link_get_state = gv100_nvlink_link_get_state,
731 .link_set_mode = gv100_nvlink_link_set_mode,
732 .link_get_mode = gv100_nvlink_link_get_mode,
733 .get_sublink_mode = gv100_nvlink_link_get_sublink_mode,
734 .get_tx_sublink_state = gv100_nvlink_link_get_tx_sublink_state,
735 .get_rx_sublink_state = gv100_nvlink_link_get_rx_sublink_state,
736 .set_sublink_mode = gv100_nvlink_link_set_sublink_mode,
737 .interface_init = gv100_nvlink_interface_init,
738 .reg_init = gv100_nvlink_reg_init,
739 .shutdown = gv100_nvlink_shutdown,
740 .early_init = gv100_nvlink_early_init,
741 },
720 .chip_init_gpu_characteristics = gv100_init_gpu_characteristics, 742 .chip_init_gpu_characteristics = gv100_init_gpu_characteristics,
721 .get_litter_value = gv100_get_litter_value, 743 .get_litter_value = gv100_get_litter_value,
722}; 744};
@@ -751,6 +773,7 @@ int gv100_init_hal(struct gk20a *g)
751 gops->xve = gv100_ops.xve; 773 gops->xve = gv100_ops.xve;
752 gops->falcon = gv100_ops.falcon; 774 gops->falcon = gv100_ops.falcon;
753 gops->priv_ring = gv100_ops.priv_ring; 775 gops->priv_ring = gv100_ops.priv_ring;
776 gops->nvlink = gv100_ops.nvlink;
754 777
755 /* clocks */ 778 /* clocks */
756 gops->clk.init_clk_support = gv100_ops.clk.init_clk_support; 779 gops->clk.init_clk_support = gv100_ops.clk.init_clk_support;
diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.c b/drivers/gpu/nvgpu/gv100/mc_gv100.c
new file mode 100644
index 00000000..a3f8ab06
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/mc_gv100.c
@@ -0,0 +1,94 @@
1/*
2 * GV100 master
3 *
4 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/types.h>
26
27#include "gk20a/gk20a.h"
28
29#include "gp10b/mc_gp10b.h"
30
31#include "mc_gv100.h"
32#include "gv11b/fb_gv11b.h"
33
34#include <nvgpu/hw/gv100/hw_mc_gv100.h>
35
36void mc_gv100_intr_enable(struct gk20a *g)
37{
38 u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g);
39
40 gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING),
41 0xffffffff);
42 gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING),
43 0xffffffff);
44 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_ALL);
45
46 g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING] =
47 mc_intr_pfifo_pending_f() |
48 mc_intr_hub_pending_f() |
49 mc_intr_priv_ring_pending_f() |
50 mc_intr_pbus_pending_f() |
51 mc_intr_ltc_pending_f() |
52 mc_intr_nvlink_pending_f() |
53 eng_intr_mask;
54
55 g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] =
56 mc_intr_pfifo_pending_f()
57 | eng_intr_mask;
58
59 /* TODO: Enable PRI faults for HUB ECC err intr */
60 gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, g->mm.hub_intr_types);
61
62 gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING),
63 g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING]);
64
65 gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING),
66 g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]);
67
68}
69
70bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0)
71{
72 return ((mc_intr_0 & mc_intr_nvlink_pending_f()) ? true : false);
73}
74
75bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id)
76{
77 u32 mc_intr_0 = gk20a_readl(g, mc_intr_r(0));
78 u32 stall_intr, eng_intr_mask;
79
80 eng_intr_mask = gk20a_fifo_act_eng_interrupt_mask(g, act_eng_id);
81 if (mc_intr_0 & eng_intr_mask)
82 return true;
83
84 stall_intr = mc_intr_pfifo_pending_f() |
85 mc_intr_hub_pending_f() |
86 mc_intr_priv_ring_pending_f() |
87 mc_intr_pbus_pending_f() |
88 mc_intr_ltc_pending_f() |
89 mc_intr_nvlink_pending_f();
90 if (mc_intr_0 & stall_intr)
91 return true;
92
93 return false;
94}
diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.h b/drivers/gpu/nvgpu/gv100/mc_gv100.h
new file mode 100644
index 00000000..4aff4a36
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/mc_gv100.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef MC_GV100_H
24#define MC_GV100_H
25struct gk20a;
26
27void mc_gv100_intr_enable(struct gk20a *g);
28bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0);
29bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id);
30#endif
diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c
new file mode 100644
index 00000000..e5e65c51
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c
@@ -0,0 +1,2654 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/nvgpu_common.h>
24#include <nvgpu/bios.h>
25#include <nvgpu/firmware.h>
26#include <nvgpu/bitops.h>
27#include <nvgpu/nvlink.h>
28#include <nvgpu/enabled.h>
29
30#include "gk20a/gk20a.h"
31#include "nvlink_gv100.h"
32
33#include <nvgpu/hw/gv100/hw_falcon_gv100.h>
34#include <nvgpu/hw/gv100/hw_top_gv100.h>
35#include <nvgpu/hw/gv100/hw_nvlinkip_discovery_gv100.h>
36#include <nvgpu/hw/gv100/hw_nvlipt_gv100.h>
37#include <nvgpu/hw/gv100/hw_ioctrl_gv100.h>
38#include <nvgpu/hw/gv100/hw_minion_gv100.h>
39#include <nvgpu/hw/gv100/hw_nvl_gv100.h>
40#include <nvgpu/hw/gv100/hw_ioctrlmif_gv100.h>
41#include <nvgpu/hw/gv100/hw_trim_gv100.h>
42#include <nvgpu/hw/gv100/hw_nvtlc_gv100.h>
43
44#include <nvgpu/hw/gv100/hw_fb_gv100.h>
45#include <nvgpu/hw/gv100/hw_mc_gv100.h>
46
47#define NVLINK_PLL_ON_TIMEOUT_MS 30
48#define NVLINK_SUBLINK_TIMEOUT_MS 200
49/*
50 * The manuals are missing some useful defines
51 * we add them for now
52 */
53
54#define IPT_INTR_CONTROL_LINK(i) (nvlipt_intr_control_link0_r() + (i)*4)
55#define IPT_ERR_UC_STATUS_LINK(i) (nvlipt_err_uc_status_link0_r() + (i)*36)
56#define IPT_ERR_UC_MASK_LINK(i) (nvlipt_err_uc_mask_link0_r() + (i)*36)
57#define IPT_ERR_UC_SEVERITY_LINK(i) (nvlipt_err_uc_severity_link0_r() + (i)*36)
58#define IPT_ERR_UC_FIRST_LINK(i) (nvlipt_err_uc_first_link0_r() + (i)*36)
59#define IPT_ERR_UC_ADVISORY_LINK(i) (nvlipt_err_uc_advisory_link0_r() + (i)*36)
60#define IPT_ERR_C_STATUS_LINK(i) (nvlipt_err_c_status_link0_r() + (i)*36)
61#define IPT_ERR_C_MASK_LINK(i) (nvlipt_err_c_mask_link0_r() + (i)*36)
62#define IPT_ERR_C_FIRST_LINK(i) (nvlipt_err_c_first_link0_r() + (i)*36)
63#define IPT_ERR_CONTROL_LINK(i) (nvlipt_err_control_link0_r() + (i)*4)
64
65#define IPT_ERR_UC_ACTIVE_BITS (nvlipt_err_uc_status_link0_dlprotocol_f(1) | \
66 nvlipt_err_uc_status_link0_datapoisoned_f(1) | \
67 nvlipt_err_uc_status_link0_flowcontrol_f(1) | \
68 nvlipt_err_uc_status_link0_responsetimeout_f(1) | \
69 nvlipt_err_uc_status_link0_targeterror_f(1) | \
70 nvlipt_err_uc_status_link0_unexpectedresponse_f(1) | \
71 nvlipt_err_uc_status_link0_receiveroverflow_f(1) | \
72 nvlipt_err_uc_status_link0_malformedpacket_f(1) | \
73 nvlipt_err_uc_status_link0_stompedpacketreceived_f(1) | \
74 nvlipt_err_uc_status_link0_unsupportedrequest_f(1) | \
75 nvlipt_err_uc_status_link0_ucinternal_f(1))
76
77
78#define MINION_FALCON_INTR_MASK (minion_falcon_irqmset_wdtmr_set_f() | \
79 minion_falcon_irqmset_halt_set_f() | \
80 minion_falcon_irqmset_exterr_set_f()| \
81 minion_falcon_irqmset_swgen0_set_f()| \
82 minion_falcon_irqmset_swgen1_set_f())
83
84#define MINION_FALCON_INTR_DEST ( \
85 minion_falcon_irqdest_host_wdtmr_host_f() | \
86 minion_falcon_irqdest_host_halt_host_f() | \
87 minion_falcon_irqdest_host_exterr_host_f() | \
88 minion_falcon_irqdest_host_swgen0_host_f() | \
89 minion_falcon_irqdest_host_swgen1_host_f() | \
90 minion_falcon_irqdest_target_wdtmr_host_normal_f() | \
91 minion_falcon_irqdest_target_halt_host_normal_f() | \
92 minion_falcon_irqdest_target_exterr_host_normal_f() | \
93 minion_falcon_irqdest_target_swgen0_host_normal_f() | \
94 minion_falcon_irqdest_target_swgen1_host_normal_f())
95
96/* Minion FW header format */
97union gv100_minion_hdr {
98 struct {
99 u32 os_code_offset;
100 u32 os_code_size;
101 u32 os_data_offset;
102 u32 os_data_size;
103 u32 num_apps;
104 u32 app_0_code_start;
105 };
106 u8 raw_data[256];
107};
108
109struct __nvlink_reginit {
110 u32 addr;
111 u32 value;
112};
113
114static const struct __nvlink_reginit __nvlink_reginit_per_link_tegra[] = {
115 /* NVTLC when connected to Tegra */
116 { 0x300U, 0x00800040U },
117 { 0x304U, 0x00000020U },
118 { 0x308U, 0x00000020U },
119 { 0x30CU, 0x00000020U },
120 { 0x310U, 0x00000020U },
121 { 0x314U, 0x00800040U },
122 { 0x318U, 0x00000000U },
123 { 0x31CU, 0x00000000U },
124 { 0x200U, 0x007F003FU },
125 { 0x204U, 0x007F005FU },
126 { 0x208U, 0x007F007FU },
127 { 0x20CU, 0x007F009FU },
128 { 0x210U, 0x007F00BFU },
129 { 0x214U, 0x00FF003FU },
130 { 0x218U, 0x00FF003FU },
131 { 0x21CU, 0x00FF003FU },
132 { 0xB00U, 0x010000BEU },
133 { 0xB04U, 0x00000064U },
134 { 0xB08U, 0x00000000U },
135 { 0xB0CU, 0x00000020U },
136 { 0xB10U, 0x00000000U },
137 { 0xB14U, 0x010000BEU },
138 { 0xB18U, 0x00000000U },
139 { 0xB1CU, 0x00000000U },
140 { 0xA00U, 0x00FF00BDU },
141 { 0xA04U, 0x00FF0121U },
142 { 0xA08U, 0x00FF0121U },
143 { 0xA0CU, 0x00FF0141U },
144 { 0xA10U, 0x00FF0141U },
145 { 0xA14U, 0x01FF01FFU },
146 { 0xA18U, 0x01FF01FFU },
147 { 0xA1CU, 0x01FF01FFU },
148 { 0xF04U, 0x00FFFFFFU },
149 { 0xF0CU, 0x00FFFFFFU },
150 { 0xF1CU, 0x003FFFFFU },
151 { 0xF24U, 0x003FFFFFU },
152 { 0x704U, 0x003FFFFFU },
153 { 0x70CU, 0x003FFFFFU },
154 { 0x400U, 0x00000001U },
155 { 0xC00U, 0x00000001U },
156};
157
158static const struct __nvlink_reginit __nvlink_reginit_per_link_gpu[] = {
159 /* NVTLC for PEER GPU */
160 { 0x300U, 0x00800040U },
161 { 0x304U, 0x00000000U },
162 { 0x308U, 0x00000000U },
163 { 0x30CU, 0x00000000U },
164 { 0x310U, 0x00000000U },
165 { 0x314U, 0x00800040U },
166 { 0x318U, 0x00000000U },
167 { 0x31CU, 0x00000000U },
168 { 0x200U, 0x007F003FU },
169 { 0x204U, 0x007F003FU },
170 { 0x208U, 0x007F003FU },
171 { 0x20CU, 0x007F003FU },
172 { 0x210U, 0x007F003FU },
173 { 0x214U, 0x00FF007FU },
174 { 0x218U, 0x00FF007FU },
175 { 0x21CU, 0x00FF007FU },
176 { 0xB00U, 0x010000C0U },
177 { 0xB04U, 0x00000000U },
178 { 0xB08U, 0x00000000U },
179 { 0xB0CU, 0x00000000U },
180 { 0xB10U, 0x00000000U },
181 { 0xB14U, 0x010000C0U },
182 { 0xB18U, 0x00000000U },
183 { 0xB1CU, 0x00000000U },
184 { 0xA00U, 0x00FF00BFU },
185 { 0xA04U, 0x00FF00BFU },
186 { 0xA08U, 0x00FF00BFU },
187 { 0xA0CU, 0x00FF00BFU },
188 { 0xA10U, 0x00FF00BFU },
189 { 0xA14U, 0x01FF017FU },
190 { 0xA18U, 0x01FF017FU },
191 { 0xA1CU, 0x01FF017FU },
192 { 0xF04U, 0x00FFFFFFU },
193 { 0xF0CU, 0x00FFFFFFU },
194 { 0xF1CU, 0x003FFFFFU },
195 { 0xF24U, 0x003FFFFFU },
196 { 0x704U, 0x003FFFFFU },
197 { 0x70CU, 0x003FFFFFU },
198 { 0x400U, 0x00000001U },
199 { 0xC00U, 0x00000001U },
200};
201
202
203#define NVL_DEVICE(str) nvlinkip_discovery_common_device_##str##_v()
204
205const char * __gv100_device_type_to_str(u32 type)
206{
207 if (type == NVL_DEVICE(ioctrl))
208 return "IOCTRL";
209 if (type == NVL_DEVICE(dlpl))
210 return "DL/PL";
211 if (type == NVL_DEVICE(nvltlc))
212 return "NVLTLC";
213 if (type == NVL_DEVICE(ioctrlmif))
214 return "IOCTRLMIF";
215 if (type == NVL_DEVICE(nvlipt))
216 return "NVLIPT";
217 if (type == NVL_DEVICE(minion))
218 return "MINION";
219 if (type == NVL_DEVICE(dlpl_multicast))
220 return "DL/PL MULTICAST";
221 if (type == NVL_DEVICE(nvltlc_multicast))
222 return "NVLTLC MULTICAST";
223 if (type == NVL_DEVICE(ioctrlmif_multicast))
224 return "IOCTRLMIF MULTICAST";
225 if (type == NVL_DEVICE(nvltlc_multicast))
226 return "NVLTLC MULTICAST";
227 return "UNKNOWN";
228}
229
230/*
231 * Function prototypes
232 */
233static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask);
234static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g);
235static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask);
236static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g,
237 unsigned long mask, bool sync);
238
239
240/*
241 *******************************************************************************
242 * IP specific functions *
243 *******************************************************************************
244 */
245
246/*
247 *-----------------------------------------------------------------------------*
248 * MINION API
249 *-----------------------------------------------------------------------------*
250 */
251
252/*
253 * Initialization of link specific interrupts
254 */
255static void gv100_nvlink_minion_link_intr_enable(struct gk20a *g, u32 link_id,
256 bool enable)
257{
258 u32 intr, links;
259
260 /* Only stall interrupts for now */
261 intr = MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
262 links = minion_minion_intr_stall_en_link_v(intr);
263
264 if (enable)
265 links |= BIT(link_id);
266 else
267 links &= ~BIT(link_id);
268
269 intr = set_field(intr, minion_minion_intr_stall_en_link_m(),
270 minion_minion_intr_stall_en_link_f(links));
271 MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), intr);
272}
273
274/*
275 * Initialization of falcon interrupts
276 */
277static void gv100_nvlink_minion_falcon_intr_enable(struct gk20a *g, bool enable)
278{
279 u32 reg;
280
281 reg = MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
282 if (enable) {
283 reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(),
284 minion_minion_intr_stall_en_fatal_enable_f());
285 reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(),
286 minion_minion_intr_stall_en_nonfatal_enable_f());
287 reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(),
288 minion_minion_intr_stall_en_falcon_stall_enable_f());
289 reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(),
290 minion_minion_intr_stall_en_falcon_nostall_enable_f());
291 } else {
292 reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(),
293 minion_minion_intr_stall_en_fatal_disable_f());
294 reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(),
295 minion_minion_intr_stall_en_nonfatal_disable_f());
296 reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(),
297 minion_minion_intr_stall_en_falcon_stall_disable_f());
298 reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(),
299 minion_minion_intr_stall_en_falcon_nostall_disable_f());
300 }
301
302 MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), reg);
303}
304
305/*
306 * Initialize minion IP interrupts
307 */
308static void gv100_nvlink_initialize_minion(struct gk20a *g)
309{
310 /* Disable non-stall tree */
311 MINION_REG_WR32(g, minion_minion_intr_nonstall_en_r(), 0x0);
312
313 gv100_nvlink_minion_falcon_intr_enable(g, true);
314}
315
316/*
317 * Check if minion is up
318 */
319static bool __gv100_nvlink_minion_is_running(struct gk20a *g)
320{
321
322 /* if minion is booted and not halted, it is running */
323 if ((MINION_REG_RD32(g, minion_minion_status_r()) &
324 minion_minion_status_status_f(1)) &&
325 (!minion_falcon_irqstat_halt_v(
326 MINION_REG_RD32(g, minion_falcon_irqstat_r()))))
327 return true;
328
329 return false;
330}
331
332/*
333 * Falcon specific ISR handling
334 */
335static bool gv100_nvlink_minion_falcon_isr(struct gk20a *g)
336{
337 u32 intr;
338
339 intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) &
340 MINION_REG_RD32(g, minion_falcon_irqmask_r());
341
342 if (!intr)
343 return true;
344
345 if (intr & falcon_falcon_irqstat_exterr_true_f()) {
346 nvgpu_err(g, "FALCON EXT ADDR: 0x%x 0x%x 0x%x",
347 MINION_REG_RD32(g, 0x244),
348 MINION_REG_RD32(g, 0x248),
349 MINION_REG_RD32(g, 0x24c));
350 }
351
352 MINION_REG_WR32(g, minion_falcon_irqsclr_r(), intr);
353
354 nvgpu_err(g, "FATAL minion IRQ: 0x%08x", intr);
355
356
357 intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) &
358 MINION_REG_RD32(g, minion_falcon_irqmask_r());
359
360 return (intr == 0);
361}
362
363/*
364 * Link Specific ISR
365 */
366
367static bool gv100_nvlink_minion_link_isr(struct gk20a *g, u32 link_id)
368{
369 u32 intr, code;
370 bool fatal = false;
371
372 intr = MINION_REG_RD32(g, minion_nvlink_link_intr_r(link_id));
373 code = minion_nvlink_link_intr_code_v(intr);
374
375 if (code == minion_nvlink_link_intr_code_swreq_v()) {
376 nvgpu_err(g, " Intr SWREQ, link: %d subcode: %x",
377 link_id, minion_nvlink_link_intr_subcode_v(intr));
378 } else if (code == minion_nvlink_link_intr_code_pmdisabled_v()) {
379 nvgpu_err(g, " Fatal Intr PMDISABLED, link: %d subcode: %x",
380 link_id, minion_nvlink_link_intr_subcode_v(intr));
381 fatal = true;
382 } else if (code == minion_nvlink_link_intr_code_na_v()) {
383 nvgpu_err(g, " Fatal Intr NA, link: %d subcode: %x",
384 link_id, minion_nvlink_link_intr_subcode_v(intr));
385 fatal = true;
386 } else if (code == minion_nvlink_link_intr_code_dlreq_v()) {
387 nvgpu_err(g, " Fatal Intr DLREQ, link: %d subcode: %x",
388 link_id, minion_nvlink_link_intr_subcode_v(intr));
389 fatal = true;
390 } else {
391 nvgpu_err(g, " Fatal Intr UNKN:%x, link: %d subcode: %x", code,
392 link_id, minion_nvlink_link_intr_subcode_v(intr));
393 fatal = true;
394 }
395
396 if (fatal)
397 gv100_nvlink_minion_link_intr_enable(g, link_id, false);
398
399 intr = set_field(intr, minion_nvlink_link_intr_state_m(),
400 minion_nvlink_link_intr_state_f(1));
401 MINION_REG_WR32(g, minion_nvlink_link_intr_r(link_id), intr);
402
403 return true;
404}
405
406/*
407 * Global minion routine to service interrupts
408 */
409static bool gv100_nvlink_minion_isr(struct gk20a *g) {
410
411 u32 intr, i;
412 unsigned long links;
413
414 intr = MINION_REG_RD32(g, minion_minion_intr_r()) &
415 MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
416
417 if (minion_minion_intr_falcon_stall_v(intr) ||
418 minion_minion_intr_falcon_nostall_v(intr))
419 gv100_nvlink_minion_falcon_isr(g);
420
421 if (minion_minion_intr_fatal_v(intr)) {
422 gv100_nvlink_minion_falcon_intr_enable(g, false);
423 MINION_REG_WR32(g, minion_minion_intr_r(),
424 minion_minion_intr_fatal_f(1));
425 }
426
427 if (minion_minion_intr_nonfatal_v(intr))
428 MINION_REG_WR32(g, minion_minion_intr_r(),
429 minion_minion_intr_nonfatal_f(1));
430
431 links = minion_minion_intr_link_v(intr) & g->nvlink.enabled_links;
432
433 if (links)
434 for_each_set_bit(i, &links, 32)
435 gv100_nvlink_minion_link_isr(g, i);
436
437 /* Re-test interrupt status */
438 intr = MINION_REG_RD32(g, minion_minion_intr_r()) &
439 MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
440
441 return (intr == 0);
442}
443
444/*
445 * Load minion FW and set up bootstrap
446 */
447static u32 gv100_nvlink_minion_load(struct gk20a *g)
448{
449 struct bin_hdr *hdr = NULL;
450 struct nvgpu_firmware *minion_fw = NULL;
451 union gv100_minion_hdr *minion_hdr;
452 u32 *minion_ucode = NULL;
453 u32 err = 0;
454 struct nvgpu_timeout timeout;
455 u32 delay = GR_IDLE_CHECK_DEFAULT;
456 u32 reg;
457 u32 app;
458
459 nvgpu_log_fn(g, " ");
460
461 if (__gv100_nvlink_minion_is_running(g))
462 return 0;
463
464 /* get mem unlock ucode binary */
465 minion_fw = nvgpu_request_firmware(g, "minion.bin", 0);
466 if (!minion_fw) {
467 nvgpu_err(g, "minion ucode get fail");
468 err = -ENOENT;
469 goto exit;
470 }
471
472 /* nvdec falcon reset */
473 nvgpu_flcn_reset(&g->minion_flcn);
474
475 hdr = (struct bin_hdr *) minion_fw->data;
476
477 minion_hdr = (union gv100_minion_hdr *) (minion_fw->data +
478 hdr->header_offset);
479 minion_ucode = (u32 *) (minion_fw->data + hdr->data_offset);
480
481 nvgpu_log(g, gpu_dbg_nvlink,
482 "os_code_offset: 0x%x", minion_hdr->os_code_offset);
483 nvgpu_log(g, gpu_dbg_nvlink,
484 "os_code_size: 0x%x", minion_hdr->os_code_size);
485 nvgpu_log(g, gpu_dbg_nvlink,
486 "os_data_offset: 0x%x", minion_hdr->os_data_offset);
487 nvgpu_log(g, gpu_dbg_nvlink,
488 "os_data_size: 0x%x", minion_hdr->os_data_size);
489 nvgpu_log(g, gpu_dbg_nvlink,
490 "num_apps: 0x%x", minion_hdr->num_apps);
491 nvgpu_log(g, gpu_dbg_nvlink,
492 "app_0_code_start: 0x%x", minion_hdr->app_0_code_start);
493
494 /* Clear interrupts */
495 nvgpu_flcn_set_irq(&g->minion_flcn, true, MINION_FALCON_INTR_MASK,
496 MINION_FALCON_INTR_DEST);
497
498 /* Copy Non Secure IMEM code */
499 nvgpu_flcn_copy_to_imem(&g->minion_flcn, 0,
500 (u8 *)&minion_ucode[minion_hdr->os_code_offset >> 2],
501 minion_hdr->os_code_size, 0, false,
502 GET_IMEM_TAG(minion_hdr->os_code_offset));
503
504 /* Copy Non Secure DMEM code */
505 nvgpu_flcn_copy_to_dmem(&g->minion_flcn, 0,
506 (u8 *)&minion_ucode[minion_hdr->os_data_offset >> 2],
507 minion_hdr->os_data_size, 0);
508
509 /* If apps are there load them securely */
510 if (minion_hdr->num_apps) {
511 for (app = 0; app < minion_hdr->num_apps; app++) {
512 u32 app_code_start, app_code_size;
513 u32 app_data_start, app_data_size;
514
515 app_code_start =
516 *((u32 *) &minion_hdr->app_0_code_start +
517 2*app);
518 app_code_size =
519 *((u32 *) &minion_hdr->app_0_code_start +
520 2*app + 1);
521 app_data_start =
522 *((u32 *) &minion_hdr->app_0_code_start +
523 2*minion_hdr->num_apps + 2*app);
524 app_data_size =
525 *((u32 *) &minion_hdr->app_0_code_start +
526 2*minion_hdr->num_apps + 2*app + 1);
527
528 nvgpu_log(g, gpu_dbg_nvlink,
529 " app %d code_offset 0x%x", app, app_code_start);
530 nvgpu_log(g, gpu_dbg_nvlink,
531 " app %d code_size 0x%x", app, app_code_size);
532 nvgpu_log(g, gpu_dbg_nvlink,
533 " app %d data_offset 0x%x", app, app_data_start);
534 nvgpu_log(g, gpu_dbg_nvlink,
535 " app %d data_size 0x%x", app, app_data_size);
536
537 if (app_code_size)
538 nvgpu_flcn_copy_to_imem(&g->minion_flcn,
539 app_code_start,
540 (u8 *)&minion_ucode[
541 app_code_start >> 2],
542 app_code_size, 0, true,
543 GET_IMEM_TAG(app_code_start));
544
545 if (app_data_size)
546 nvgpu_flcn_copy_to_dmem(&g->minion_flcn,
547 app_data_start,
548 (u8 *)&minion_ucode[
549 app_data_start >> 2],
550 app_data_size, 0);
551 }
552 }
553
554 /* set BOOTVEC to start of non-secure code */
555 nvgpu_flcn_bootstrap(&g->minion_flcn, 0x0);
556
557 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
558 NVGPU_TIMER_CPU_TIMER);
559
560 do {
561 reg = MINION_REG_RD32(g, minion_minion_status_r());
562
563 if (minion_minion_status_status_v(reg)) {
564 /* Minion sequence completed, check status */
565 if (minion_minion_status_status_v(reg) !=
566 minion_minion_status_status_boot_v()) {
567 nvgpu_err(g, "MINION init sequence failed: 0x%x",
568 minion_minion_status_status_v(reg));
569 err = -EINVAL;
570
571 goto exit;
572 }
573
574 nvgpu_log(g, gpu_dbg_nvlink,
575 "MINION boot successful: 0x%x", reg);
576 err = 0;
577 break;
578 }
579
580 nvgpu_usleep_range(delay, delay * 2);
581 delay = min_t(unsigned long,
582 delay << 1, GR_IDLE_CHECK_MAX);
583 } while (!nvgpu_timeout_expired_msg(&timeout, " minion boot timeout"));
584
585 /* Service interrupts */
586 gv100_nvlink_minion_falcon_isr(g);
587
588 if (nvgpu_timeout_peek_expired(&timeout)) {
589 err = -ETIMEDOUT;
590 goto exit;
591 }
592
593 gv100_nvlink_initialize_minion(g);
594
595exit:
596 if (minion_fw)
597 nvgpu_release_firmware(g, minion_fw);
598
599 return err;
600}
601
602/*
603 * Check if MINION command is complete
604 */
605static u32 gv100_nvlink_minion_command_complete(struct gk20a *g, u32 link_id)
606{
607 u32 reg;
608 struct nvgpu_timeout timeout;
609 u32 delay = GR_IDLE_CHECK_DEFAULT;
610
611
612 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
613 NVGPU_TIMER_CPU_TIMER);
614
615 do {
616 reg = MINION_REG_RD32(g, minion_nvlink_dl_cmd_r(link_id));
617
618 if (minion_nvlink_dl_cmd_ready_v(reg) == 1) {
619 /* Command completed, check sucess */
620 if (minion_nvlink_dl_cmd_fault_v(reg) == 1) {
621 nvgpu_err(g, "minion cmd(%d) error: 0x%x",
622 link_id, reg);
623
624 reg = minion_nvlink_dl_cmd_fault_f(1);
625 MINION_REG_WR32(g,
626 minion_nvlink_dl_cmd_r(link_id), reg);
627
628 return -EINVAL;
629 }
630
631 /* Commnand success */
632 break;
633 }
634 nvgpu_usleep_range(delay, delay * 2);
635 delay = min_t(unsigned long,
636 delay << 1, GR_IDLE_CHECK_MAX);
637
638 } while (!nvgpu_timeout_expired_msg(&timeout, " minion cmd timeout"));
639
640 if (nvgpu_timeout_peek_expired(&timeout))
641 return -ETIMEDOUT;
642
643 nvgpu_log(g, gpu_dbg_nvlink, "minion cmd Complete");
644 return 0;
645}
646
647/*
648 * Send Minion command (can be async)
649 */
650static u32 gv100_nvlink_minion_send_command(struct gk20a *g,
651 u32 link_id, u32 command, u32 scratch_0, bool sync)
652{
653 u32 err = 0;
654
655 /* Check last command succeded */
656 err = gv100_nvlink_minion_command_complete(g, link_id);
657 if (err)
658 return -EINVAL;
659
660 nvgpu_log(g, gpu_dbg_nvlink,
661 "sending MINION command 0x%x to link %d", command, link_id);
662
663 if (command == minion_nvlink_dl_cmd_command_configeom_v())
664 MINION_REG_WR32(g, minion_misc_0_r(),
665 minion_misc_0_scratch_swrw_0_f(scratch_0));
666
667 MINION_REG_WR32(g, minion_nvlink_dl_cmd_r(link_id),
668 minion_nvlink_dl_cmd_command_f(command) |
669 minion_nvlink_dl_cmd_fault_f(1));
670
671 if (sync)
672 err = gv100_nvlink_minion_command_complete(g, link_id);
673
674 return err;
675}
676
677/* MINION API COMMANDS */
678
679/*
680 * Init UPHY
681 */
682static u32 gv100_nvlink_minion_init_uphy(struct gk20a *g, unsigned long mask,
683 bool sync)
684{
685 u32 err = 0;
686 u32 init_pll_cmd;
687 u32 link_id, master_pll, slave_pll;
688 u32 master_state, slave_state;
689
690 unsigned long link_enable;
691
692 switch(g->nvlink.speed) {
693 case nvgpu_nvlink_speed_default:
694 init_pll_cmd = minion_nvlink_dl_cmd_command_initpll_0_v();
695 break;
696 default:
697 nvgpu_err(g, "Unsupported UPHY speed");
698 return -EINVAL;
699 }
700
701 link_enable = __gv100_nvlink_get_link_reset_mask(g);
702
703 /* Cannot use links 0/1 without refclk buffer */
704 if (mask & (BIT(1)|BIT(0))) {
705 nvgpu_err(g, "links 0/1 not supported on GV100");
706 return -EINVAL;
707 }
708
709 for_each_set_bit(link_id, &mask, 32) {
710 master_pll = g->nvlink.links[link_id].pll_master_link_id;
711 slave_pll = g->nvlink.links[link_id].pll_slave_link_id;
712
713 master_state = nvl_link_state_state_init_v();
714 slave_state = nvl_link_state_state_init_v();
715
716 if (BIT(master_pll) & link_enable)
717 master_state = nvl_link_state_state_v(
718 g->ops.nvlink.link_get_state(g, master_pll));
719
720 if (BIT(slave_pll) & link_enable)
721 slave_state = nvl_link_state_state_v(
722 g->ops.nvlink.link_get_state(g, slave_pll));
723
724 if ((slave_state != nvl_link_state_state_init_v()) ||
725 (master_state != nvl_link_state_state_init_v())) {
726 nvgpu_err(g, "INIT PLL can only be executed when both "
727 "master and slave links are in init state");
728 return -EINVAL;
729 }
730
731 /* Check if INIT PLL is done on link */
732 if (!(BIT(master_pll) & g->nvlink.init_pll_done)) {
733 err = gv100_nvlink_minion_send_command(g, master_pll,
734 init_pll_cmd, 0, sync);
735 if (err) {
736 nvgpu_err(g, " Error sending INITPLL to minion");
737 return err;
738 }
739
740 g->nvlink.init_pll_done |= BIT(master_pll);
741 }
742 }
743
744 err = gv100_nvlink_setup_pll(g, mask);
745 if (err) {
746 nvgpu_err(g, "Error setting up PLL");
747 return err;
748 }
749
750 /* INITPHY commands */
751 for_each_set_bit(link_id, &mask, 32) {
752 err = gv100_nvlink_minion_send_command(g, link_id,
753 minion_nvlink_dl_cmd_command_initphy_v(), 0, sync);
754 if (err) {
755 nvgpu_err(g, "Error on INITPHY minion DL command %u",
756 link_id);
757 return err;
758 }
759 }
760
761 return 0;
762}
763
764/*
765 * Configure AC coupling
766 */
767static u32 gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g,
768 unsigned long mask, bool sync)
769{
770 u32 err = 0;
771 u32 i;
772 u32 temp;
773
774 for_each_set_bit(i, &mask, 32) {
775
776 temp = DLPL_REG_RD32(g, i, nvl_link_config_r());
777 temp &= ~nvl_link_config_ac_safe_en_m();
778 temp |= nvl_link_config_ac_safe_en_on_f();
779
780 DLPL_REG_WR32(g, i, nvl_link_config_r(), temp);
781
782 err = gv100_nvlink_minion_send_command(g, i,
783 minion_nvlink_dl_cmd_command_setacmode_v(), 0, sync);
784
785 if (err)
786 return err;
787 }
788
789 return err;
790}
791
792/*
793 * Set Data ready
794 */
795static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g,
796 unsigned long mask, bool sync)
797{
798 u32 err = 0;
799 u32 i;
800
801 for_each_set_bit(i, &mask, 32) {
802 err = gv100_nvlink_minion_send_command(g, i,
803 minion_nvlink_dl_cmd_command_initlaneenable_v(), 0,
804 sync);
805 if (err) {
806 nvgpu_err(g, "Failed init lane enable on minion");
807 return err;
808 }
809 }
810
811 for_each_set_bit(i, &mask, 32) {
812 err = gv100_nvlink_minion_send_command(g, i,
813 minion_nvlink_dl_cmd_command_initdlpl_v(), 0, sync);
814 if (err) {
815 nvgpu_err(g, "Failed init DLPL on minion");
816 return err;
817 }
818 }
819 return err;
820}
821
822/*
823 * Request that minion disable the lane
824 */
825static u32 gv100_nvlink_minion_lane_disable(struct gk20a *g, u32 link_id,
826 bool sync)
827{
828 u32 err = 0;
829
830 err = gv100_nvlink_minion_send_command(g, link_id,
831 minion_nvlink_dl_cmd_command_lanedisable_v(), 0, sync);
832
833 if (err)
834 nvgpu_err(g, " failed to disable lane on %d", link_id);
835
836 return err;
837}
838
839/*
840 * Request that minion shutdown the lane
841 */
842static u32 gv100_nvlink_minion_lane_shutdown(struct gk20a *g, u32 link_id,
843 bool sync)
844{
845 u32 err = 0;
846
847 err = gv100_nvlink_minion_send_command(g, link_id,
848 minion_nvlink_dl_cmd_command_laneshutdown_v(), 0, sync);
849
850 if (err)
851 nvgpu_err(g, " failed to shutdown lane on %d", link_id);
852
853 return err;
854}
855
856
857
858/*
859 *-----------------------------------------------------------------------------*
860 * TLC API
861 *-----------------------------------------------------------------------------*
862 */
863
864static int gv100_nvlink_get_tlc_reginit(enum nvgpu_nvlink_endp endp,
865 struct __nvlink_reginit **reg, u32 *count)
866{
867 switch(endp) {
868 case nvgpu_nvlink_endp_tegra:
869 *reg = (struct __nvlink_reginit *)
870 __nvlink_reginit_per_link_tegra;
871 *count = ARRAY_SIZE(__nvlink_reginit_per_link_tegra);
872 break;
873 case nvgpu_nvlink_endp_gpu:
874 *reg = (struct __nvlink_reginit *)
875 __nvlink_reginit_per_link_gpu;
876 *count = ARRAY_SIZE(__nvlink_reginit_per_link_gpu);
877 break;
878 default:
879 return -EINVAL;
880 }
881
882 return 0;
883}
884
885/*
886 * Init TLC IP and prod regs
887 */
888static void gv100_nvlink_initialize_tlc(struct gk20a *g, u32 link_id)
889{
890}
891
892/*
893 * Init TLC per link interrupts
894 */
895static void gv100_nvlink_tlc_intr_enable(struct gk20a *g, u32 link_id,
896 bool enable)
897{
898 u32 reg_rx0 = 0, reg_rx1 = 0, reg_tx = 0;
899
900 if (enable) {
901 /* Set PROD values */
902 reg_rx0 = 0x0FFFFFF;
903 reg_rx1 = 0x03FFFFF;
904 reg_tx = 0x1FFFFFF;
905 }
906
907 TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_0_r(), reg_rx0);
908 TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_1_r(), reg_rx1);
909 TLC_REG_WR32(g, link_id, nvtlc_tx_err_report_en_0_r(), reg_tx);
910}
911
912/*
913 * helper function to get TLC intr status in common structure
914 */
915static void gv100_nvlink_tlc_get_intr(struct gk20a *g, u32 link_id)
916{
917 g->nvlink.tlc_rx_err_status_0[link_id] =
918 TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_0_r());
919 g->nvlink.tlc_rx_err_status_1[link_id] =
920 TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_1_r());
921 g->nvlink.tlc_tx_err_status_0[link_id] =
922 TLC_REG_RD32(g, link_id, nvtlc_tx_err_status_0_r());
923}
924
925/*
926 * Interrupt routine handler for TLC
927 */
928static void gv100_nvlink_tlc_isr(struct gk20a *g, u32 link_id)
929{
930
931 if (g->nvlink.tlc_rx_err_status_0[link_id]) {
932 /* All TLC RX 0 errors are fatal. Notify and disable */
933 nvgpu_err(g, "Fatal TLC RX 0 interrupt on link %d mask: %x",
934 link_id, g->nvlink.tlc_rx_err_status_0[link_id]);
935 TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_0_r(),
936 g->nvlink.tlc_rx_err_status_0[link_id]);
937 TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_0_r(),
938 g->nvlink.tlc_rx_err_status_0[link_id]);
939 }
940 if (g->nvlink.tlc_rx_err_status_1[link_id]) {
941 /* All TLC RX 1 errors are fatal. Notify and disable */
942 nvgpu_err(g, "Fatal TLC RX 1 interrupt on link %d mask: %x",
943 link_id, g->nvlink.tlc_rx_err_status_1[link_id]);
944 TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_1_r(),
945 g->nvlink.tlc_rx_err_status_1[link_id]);
946 TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_1_r(),
947 g->nvlink.tlc_rx_err_status_1[link_id]);
948 }
949 if (g->nvlink.tlc_tx_err_status_0[link_id]) {
950 /* All TLC TX 0 errors are fatal. Notify and disable */
951 nvgpu_err(g, "Fatal TLC TX 0 interrupt on link %d mask: %x",
952 link_id, g->nvlink.tlc_tx_err_status_0[link_id]);
953 TLC_REG_WR32(g, link_id, nvtlc_tx_err_first_0_r(),
954 g->nvlink.tlc_tx_err_status_0[link_id]);
955 TLC_REG_WR32(g, link_id, nvtlc_tx_err_status_0_r(),
956 g->nvlink.tlc_tx_err_status_0[link_id]);
957 }
958}
959
960/*
961 *-----------------------------------------------------------------------------*
962 * DLPL API
963 *-----------------------------------------------------------------------------*
964 */
965
966/*
967 * DLPL interrupt enable helper
968 */
969static void gv100_nvlink_dlpl_intr_enable(struct gk20a *g, u32 link_id,
970 bool enable)
971{
972 u32 reg = 0;
973
974 /* Always disable nonstall tree */
975 DLPL_REG_WR32(g, link_id, nvl_intr_nonstall_en_r(), 0);
976
977 if (!enable)
978 {
979 DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), 0);
980 return;
981 }
982
983 /* Clear interrupt register to get rid of stale state (W1C) */
984 DLPL_REG_WR32(g, link_id, nvl_intr_r(), 0xffffffff);
985 DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff);
986
987 reg = nvl_intr_stall_en_tx_recovery_long_enable_f() |
988 nvl_intr_stall_en_tx_fault_ram_enable_f() |
989 nvl_intr_stall_en_tx_fault_interface_enable_f() |
990 nvl_intr_stall_en_tx_fault_sublink_change_enable_f() |
991 nvl_intr_stall_en_rx_fault_sublink_change_enable_f() |
992 nvl_intr_stall_en_rx_fault_dl_protocol_enable_f() |
993 nvl_intr_stall_en_ltssm_fault_enable_f();
994
995 DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), reg);
996
997 /* Configure error threshold */
998 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_error_rate_ctrl_r());
999 reg = set_field(reg, nvl_sl1_error_rate_ctrl_short_threshold_man_m(),
1000 nvl_sl1_error_rate_ctrl_short_threshold_man_f(0x2));
1001 reg = set_field(reg, nvl_sl1_error_rate_ctrl_long_threshold_man_m(),
1002 nvl_sl1_error_rate_ctrl_long_threshold_man_f(0x2));
1003 DLPL_REG_WR32(g, link_id, nvl_sl1_error_rate_ctrl_r(), reg);
1004}
1005
1006/*
1007 * DLPL per-link isr
1008 */
1009
1010#define DLPL_NON_FATAL_INTR_MASK (nvl_intr_tx_replay_f(1) | \
1011 nvl_intr_tx_recovery_short_f(1) | \
1012 nvl_intr_tx_recovery_long_f(1) | \
1013 nvl_intr_rx_short_error_rate_f(1) | \
1014 nvl_intr_rx_long_error_rate_f(1) | \
1015 nvl_intr_rx_ila_trigger_f(1) | \
1016 nvl_intr_ltssm_protocol_f(1))
1017
1018#define DLPL_FATAL_INTR_MASK ( nvl_intr_ltssm_fault_f(1) | \
1019 nvl_intr_rx_fault_dl_protocol_f(1) | \
1020 nvl_intr_rx_fault_sublink_change_f(1) | \
1021 nvl_intr_tx_fault_sublink_change_f(1) | \
1022 nvl_intr_tx_fault_interface_f(1) | \
1023 nvl_intr_tx_fault_ram_f(1))
1024
1025static void gv100_nvlink_dlpl_isr(struct gk20a *g, u32 link_id)
1026{
1027 u32 non_fatal_mask = 0;
1028 u32 fatal_mask = 0;
1029 u32 intr = 0;
1030 bool retrain = false;
1031 u32 err;
1032
1033 intr = DLPL_REG_RD32(g, link_id, nvl_intr_r()) &
1034 DLPL_REG_RD32(g, link_id, nvl_intr_stall_en_r());
1035
1036 if (!intr)
1037 return;
1038
1039 fatal_mask = intr & DLPL_FATAL_INTR_MASK;
1040 non_fatal_mask = intr & DLPL_NON_FATAL_INTR_MASK;
1041
1042 nvgpu_err(g, " handling DLPL %d isr. Fatal: %x non-Fatal: %x",
1043 link_id, fatal_mask, non_fatal_mask);
1044
1045 /* Check if we are not handling an interupt */
1046 if ((fatal_mask | non_fatal_mask) & ~intr)
1047 nvgpu_err(g, "Unable to service DLPL intr on link %d", link_id);
1048
1049 if (non_fatal_mask & nvl_intr_tx_recovery_long_f(1))
1050 retrain = true;
1051 if (fatal_mask)
1052 retrain = false;
1053
1054 if (retrain) {
1055 err = nvgpu_nvlink_train(g, link_id, false);
1056 if (err)
1057 nvgpu_err(g, "failed to retrain link %d", link_id);
1058 }
1059
1060 /* Clear interrupts */
1061 DLPL_REG_WR32(g, link_id, nvl_intr_r(), (non_fatal_mask | fatal_mask));
1062 DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff);
1063}
1064
1065/*
1066 *-----------------------------------------------------------------------------*
1067 * MIF API
1068 *-----------------------------------------------------------------------------*
1069 */
1070
1071/*
1072 * Initialize MIF API with PROD settings
1073 */
1074static void gv100_nvlink_initialize_mif(struct gk20a *g, u32 link_id)
1075{
1076 u32 tmp;
1077
1078 /* Enable MIF RX error */
1079
1080 /* Containment (make fatal) */
1081 tmp = 0;
1082 tmp = set_field(tmp,
1083 ioctrlmif_rx_err_contain_en_0_rxramdataparityerr_m(),
1084 ioctrlmif_rx_err_contain_en_0_rxramdataparityerr__prod_f());
1085 tmp = set_field(tmp,
1086 ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr_m(),
1087 ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr__prod_f());
1088 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_contain_en_0_r(), tmp);
1089
1090 /* Logging (do not ignore) */
1091 tmp = 0;
1092 tmp = set_field(tmp,
1093 ioctrlmif_rx_err_log_en_0_rxramdataparityerr_m(),
1094 ioctrlmif_rx_err_log_en_0_rxramdataparityerr_f(1));
1095 tmp = set_field(tmp,
1096 ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_m(),
1097 ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_f(1));
1098 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_log_en_0_r(), tmp);
1099
1100 /* Tx Error */
1101 /* Containment (make fatal) */
1102 tmp = 0;
1103 tmp = set_field(tmp,
1104 ioctrlmif_tx_err_contain_en_0_txramdataparityerr_m(),
1105 ioctrlmif_tx_err_contain_en_0_txramdataparityerr__prod_f());
1106 tmp = set_field(tmp,
1107 ioctrlmif_tx_err_contain_en_0_txramhdrparityerr_m(),
1108 ioctrlmif_tx_err_contain_en_0_txramhdrparityerr__prod_f());
1109 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_contain_en_0_r(), tmp);
1110
1111 /* Logging (do not ignore) */
1112 tmp = 0;
1113 tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramdataparityerr_m(),
1114 ioctrlmif_tx_err_log_en_0_txramdataparityerr_f(1));
1115 tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramhdrparityerr_m(),
1116 ioctrlmif_tx_err_log_en_0_txramhdrparityerr_f(1));
1117 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_log_en_0_r(), tmp);
1118
1119 /* Credit release */
1120 MIF_REG_WR32(g, link_id, ioctrlmif_rx_ctrl_buffer_ready_r(), 0x1);
1121 MIF_REG_WR32(g, link_id, ioctrlmif_tx_ctrl_buffer_ready_r(), 0x1);
1122}
1123
1124/*
1125 * Enable per-link MIF interrupts
1126 */
1127static void gv100_nvlink_mif_intr_enable(struct gk20a *g, u32 link_id,
1128 bool enable)
1129{
1130 u32 reg0 = 0, reg1 = 0;
1131
1132 if (enable) {
1133 reg0 = set_field(reg0,
1134 ioctrlmif_rx_err_report_en_0_rxramdataparityerr_m(),
1135 ioctrlmif_rx_err_report_en_0_rxramdataparityerr_f(1));
1136 reg0 = set_field(reg0,
1137 ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_m(),
1138 ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_f(1));
1139 reg1 = set_field(reg1,
1140 ioctrlmif_tx_err_report_en_0_txramdataparityerr_m(),
1141 ioctrlmif_tx_err_report_en_0_txramdataparityerr_f(1));
1142 reg1 = set_field(reg1,
1143 ioctrlmif_tx_err_report_en_0_txramhdrparityerr_m(),
1144 ioctrlmif_tx_err_report_en_0_txramhdrparityerr_f(1));
1145 }
1146
1147 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_report_en_0_r(), reg0);
1148 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_report_en_0_r(), reg1);
1149}
1150
1151/*
1152 * Handle per-link MIF interrupts
1153 */
1154static void gv100_nvlink_mif_isr(struct gk20a *g, u32 link_id)
1155{
1156 u32 intr, fatal_mask = 0;
1157
1158 /* RX Errors */
1159 intr = MIF_REG_RD32(g, link_id, ioctrlmif_rx_err_status_0_r());
1160 if (intr) {
1161 if (intr & ioctrlmif_rx_err_status_0_rxramdataparityerr_m()) {
1162 nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_DATA_PARITY",
1163 link_id);
1164 fatal_mask |= ioctrlmif_rx_err_status_0_rxramdataparityerr_f(1);
1165 }
1166 if (intr & ioctrlmif_rx_err_status_0_rxramhdrparityerr_m()) {
1167 nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_HDR_PARITY",
1168 link_id);
1169 fatal_mask |= ioctrlmif_rx_err_status_0_rxramhdrparityerr_f(1);
1170 }
1171
1172 if (fatal_mask) {
1173 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_first_0_r(),
1174 fatal_mask);
1175 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_status_0_r(),
1176 fatal_mask);
1177 }
1178 }
1179
1180 /* TX Errors */
1181 fatal_mask = 0;
1182 intr = MIF_REG_RD32(g, link_id, ioctrlmif_tx_err_status_0_r());
1183 if (intr) {
1184 if (intr & ioctrlmif_tx_err_status_0_txramdataparityerr_m()) {
1185 nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_DATA_PARITY",
1186 link_id);
1187 fatal_mask |= ioctrlmif_tx_err_status_0_txramdataparityerr_f(1);
1188 }
1189 if (intr & ioctrlmif_tx_err_status_0_txramhdrparityerr_m()) {
1190 nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_HDR_PARITY",
1191 link_id);
1192 fatal_mask |= ioctrlmif_tx_err_status_0_txramhdrparityerr_f(1);
1193 }
1194
1195 if (fatal_mask) {
1196 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_first_0_r(),
1197 fatal_mask);
1198 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_status_0_r(),
1199 fatal_mask);
1200 }
1201 }
1202}
1203
1204/*
1205 *-----------------------------------------------------------------------------*
1206 * NVLIPT API
1207 *-----------------------------------------------------------------------------*
1208 */
1209
1210/*
1211 * NVLIPT IP initialization (per-link)
1212 */
1213static void gv100_nvlink_initialize_nvlipt(struct gk20a *g, u32 link_id)
1214{
1215 /* init persistent scratch registers */
1216 IPT_REG_WR32(g, nvlipt_scratch_cold_r(),
1217 nvlipt_scratch_cold_data_init_v());
1218
1219 /*
1220 * AErr settings (top level)
1221 */
1222
1223 /* UC first and status reg (W1C) need to be cleared byt arch */
1224 IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1225 IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1226
1227 /* AErr Severity */
1228 IPT_REG_WR32(g, IPT_ERR_UC_SEVERITY_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1229
1230 /* AErr Control settings */
1231 IPT_REG_WR32(g, IPT_ERR_CONTROL_LINK(link_id),
1232 nvlipt_err_control_link0_fatalenable_f(1) |
1233 nvlipt_err_control_link0_nonfatalenable_f(1));
1234}
1235
1236/*
1237 * Enable NVLIPT interrupts
1238 */
1239static void gv100_nvlink_nvlipt_intr_enable(struct gk20a *g, u32 link_id,
1240 bool enable)
1241{
1242 u32 val = 0;
1243 u32 reg;
1244
1245 if (enable)
1246 val = 1;
1247
1248 reg = IPT_REG_RD32(g, IPT_INTR_CONTROL_LINK(link_id));
1249 reg = set_field(reg, nvlipt_intr_control_link0_stallenable_m(),
1250 nvlipt_intr_control_link0_stallenable_f(val));
1251 reg = set_field(reg, nvlipt_intr_control_link0_nostallenable_m(),
1252 nvlipt_intr_control_link0_nostallenable_f(val));
1253 IPT_REG_WR32(g, IPT_INTR_CONTROL_LINK(link_id), reg);
1254}
1255
1256/*
1257 * Per-link NVLIPT ISR handler
1258 */
1259bool gv100_nvlink_nvlipt_isr(struct gk20a *g, u32 link_id)
1260{
1261 /*
1262 * Interrupt handling happens in leaf handlers. Assume all interrupts
1263 * were handled and clear roll ups/
1264 */
1265 IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1266 IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1267
1268 return true;
1269}
1270
1271/*
1272 *******************************************************************************
1273 * Interrupt handling functions *
1274 *******************************************************************************
1275 */
1276
1277/*
1278 * Enable common interrupts
1279 */
1280static void gv100_nvlink_common_intr_enable(struct gk20a *g,
1281 unsigned long mask)
1282{
1283 u32 reg, i;
1284
1285 /* Init HS HUB SW state */
1286 g->nvlink.hshub_config0 = gk20a_readl(g, fb_hshub_config0_r());
1287 g->nvlink.hshub_config1 = gk20a_readl(g, fb_hshub_config1_r());
1288 g->nvlink.hshub_config2 = gk20a_readl(g, fb_hshub_config2_r());
1289 g->nvlink.hshub_config6 = gk20a_readl(g, fb_hshub_config6_r());
1290
1291 /* Init IOCTRL */
1292 for_each_set_bit(i, &mask, 32) {
1293 reg = IOCTRL_REG_RD32(g, ioctrl_link_intr_0_mask_r(i));
1294 reg |= (ioctrl_link_intr_0_mask_fatal_f(1) |
1295 ioctrl_link_intr_0_mask_nonfatal_f(1) |
1296 ioctrl_link_intr_0_mask_correctable_f(1) |
1297 ioctrl_link_intr_0_mask_intra_f(1));
1298 IOCTRL_REG_WR32(g, ioctrl_link_intr_0_mask_r(i), reg);
1299 }
1300
1301 reg = IOCTRL_REG_RD32(g, ioctrl_common_intr_0_mask_r());
1302 reg |= (ioctrl_common_intr_0_mask_fatal_f(1) |
1303 ioctrl_common_intr_0_mask_nonfatal_f(1) |
1304 ioctrl_common_intr_0_mask_correctable_f(1) |
1305 ioctrl_common_intr_0_mask_intra_f(1));
1306 IOCTRL_REG_WR32(g, ioctrl_common_intr_0_mask_r(), reg);
1307
1308 /* Init NVLIPT */
1309 IPT_REG_WR32(g, nvlipt_intr_control_common_r(),
1310 nvlipt_intr_control_common_stallenable_f(1) |
1311 nvlipt_intr_control_common_nonstallenable_f(1));
1312}
1313
1314/*
1315 * Enable link specific interrupts (top-level)
1316 */
1317static void gv100_nvlink_enable_link_intr(struct gk20a *g, u32 link_id,
1318 bool enable)
1319{
1320 gv100_nvlink_minion_link_intr_enable(g, link_id, enable);
1321 gv100_nvlink_dlpl_intr_enable(g, link_id, enable);
1322 gv100_nvlink_tlc_intr_enable(g, link_id, enable);
1323 gv100_nvlink_mif_intr_enable(g, link_id, enable);
1324 gv100_nvlink_nvlipt_intr_enable(g, link_id, enable);
1325}
1326
1327/*
1328 * Top level interrupt handler
1329 */
1330int gv100_nvlink_isr(struct gk20a *g)
1331{
1332 unsigned long links;
1333 u32 link_id;
1334
1335 links = ioctrl_top_intr_0_status_link_v(
1336 IOCTRL_REG_RD32(g, ioctrl_top_intr_0_status_r()));
1337
1338 links &= g->nvlink.enabled_links;
1339 /* As per ARCH minion must be serviced first */
1340 gv100_nvlink_minion_isr(g);
1341
1342 for_each_set_bit(link_id, &links, 32) {
1343 /* Cache error logs from TLC, DL handler may clear them */
1344 gv100_nvlink_tlc_get_intr(g, link_id);
1345 gv100_nvlink_dlpl_isr(g, link_id);
1346 gv100_nvlink_tlc_isr(g, link_id);
1347 gv100_nvlink_mif_isr(g, link_id);
1348
1349 /* NVLIPT is top-level. Do it last */
1350 gv100_nvlink_nvlipt_isr(g, link_id);
1351 }
1352 return 0;
1353}
1354
1355/*******************************************************************************
1356 * Helper functions *
1357 *******************************************************************************
1358 */
1359
1360static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g)
1361{
1362 u32 reg_data;
1363
1364 reg_data = IOCTRL_REG_RD32(g, ioctrl_reset_r());
1365
1366 return ioctrl_reset_linkreset_v(reg_data);
1367}
1368
1369static u32 __gv100_nvlink_state_load_hal(struct gk20a *g)
1370{
1371 unsigned long discovered = g->nvlink.discovered_links;
1372
1373 gv100_nvlink_common_intr_enable(g, discovered);
1374 return gv100_nvlink_minion_load(g);
1375}
1376
1377#define TRIM_SYS_NVLINK_CTRL(i) (trim_sys_nvlink0_ctrl_r() + 16*i)
1378#define TRIM_SYS_NVLINK_STATUS(i) (trim_sys_nvlink0_status_r() + 16*i)
1379
1380static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask)
1381{
1382 u32 reg;
1383 u32 i;
1384 u32 links_off;
1385 struct nvgpu_timeout timeout;
1386 u32 pad_ctrl, swap_ctrl;
1387 u32 pll_id;
1388
1389 reg = gk20a_readl(g, trim_sys_nvlink_uphy_cfg_r());
1390 reg = set_field(reg, trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_m(),
1391 trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_f(1));
1392 gk20a_writel(g, trim_sys_nvlink_uphy_cfg_r(), reg);
1393
1394 reg = gk20a_readl(g, top_nvhsclk_ctrl_r());
1395
1396 pad_ctrl = top_nvhsclk_ctrl_e_clk_nvl_v(reg);
1397 swap_ctrl = top_nvhsclk_ctrl_swap_clk_nvl_v(reg);
1398
1399 for_each_set_bit(i, &mask, 32) {
1400 /* There are 3 PLLs for 6 links. We have 3 bits for each PLL.
1401 * The PLL bit corresponding to a link is /2 of its master link.
1402 */
1403 pll_id = g->nvlink.links[i].pll_master_link_id >> 1;
1404 pad_ctrl |= BIT(pll_id);
1405 swap_ctrl |= BIT(pll_id);
1406 }
1407
1408 reg = set_field(reg, top_nvhsclk_ctrl_e_clk_nvl_m(),
1409 top_nvhsclk_ctrl_e_clk_nvl_f(pad_ctrl));
1410 reg = set_field(reg, top_nvhsclk_ctrl_e_clk_core_m(),
1411 top_nvhsclk_ctrl_e_clk_core_f(0x1));
1412 reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_nvl_m(),
1413 top_nvhsclk_ctrl_swap_clk_nvl_f(swap_ctrl));
1414 reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_core_m(),
1415 top_nvhsclk_ctrl_swap_clk_core_f(0x0));
1416
1417 gk20a_writel(g, top_nvhsclk_ctrl_r(), reg);
1418
1419 for_each_set_bit(i, &mask, 32) {
1420 reg = gk20a_readl(g, TRIM_SYS_NVLINK_CTRL(i));
1421 reg = set_field(reg,
1422 trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_m(),
1423 trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_f(0));
1424 gk20a_writel(g, TRIM_SYS_NVLINK_CTRL(i), reg);
1425 }
1426
1427 /* Poll for links to go up */
1428 links_off = mask;
1429
1430 nvgpu_timeout_init(g, &timeout,
1431 NVLINK_PLL_ON_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
1432 do {
1433 for_each_set_bit(i, &mask, 32) {
1434 reg = gk20a_readl(g, TRIM_SYS_NVLINK_STATUS(i));
1435 if (trim_sys_nvlink0_status_pll_off_v(reg) == 0)
1436 links_off &= ~BIT(i);
1437 }
1438 nvgpu_udelay(5);
1439
1440 } while((!nvgpu_timeout_expired_msg(&timeout, "timeout on pll on")) &&
1441 links_off);
1442
1443 if (nvgpu_timeout_peek_expired(&timeout))
1444 return -ETIMEDOUT;
1445
1446 return 0;
1447}
1448
1449static void gv100_nvlink_prog_alt_clk(struct gk20a *g)
1450{
1451 u32 tmp;
1452
1453 /* RMW registers need to be separate */
1454 tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r());
1455 tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m();
1456 tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f();
1457 gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp);
1458
1459 tmp &= ~trim_sys_nvl_common_clk_alt_switch_finalsel_m();
1460 tmp |= trim_sys_nvl_common_clk_alt_switch_finalsel_slowclk_f();
1461}
1462
1463static int gv100_nvlink_enable_links_pre_top(struct gk20a *g, u32 links)
1464{
1465 u32 link_id;
1466 unsigned long enabled_links = links;
1467 u32 tmp;
1468 u32 reg;
1469 u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v();
1470 u32 err;
1471
1472 nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", enabled_links);
1473 /* Take links out of reset */
1474 for_each_set_bit(link_id, &enabled_links, 32) {
1475 reg = IOCTRL_REG_RD32(g, ioctrl_reset_r());
1476
1477 tmp = (BIT(link_id) |
1478 BIT(g->nvlink.links[link_id].pll_master_link_id));
1479
1480 reg = set_field(reg, ioctrl_reset_linkreset_m(),
1481 ioctrl_reset_linkreset_f( ioctrl_reset_linkreset_v(reg) |
1482 tmp));
1483
1484 IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg);
1485 nvgpu_udelay(delay);
1486
1487 reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r());
1488
1489 reg &= ~ioctrl_debug_reset_link_f(BIT(link_id));
1490 IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
1491 nvgpu_udelay(delay);
1492
1493 reg |= ioctrl_debug_reset_link_f(BIT(link_id));
1494 IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
1495 nvgpu_udelay(delay);
1496
1497 /* Enable Link DLPL for AN0 */
1498 reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r());
1499 reg = set_field(reg, nvl_link_config_link_en_m(),
1500 nvl_link_config_link_en_f(1));
1501 DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg);
1502
1503 /* This should be done by the NVLINK API */
1504 err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true);
1505 if (err) {
1506 nvgpu_err(g, "Failed to init phy of link: %u", link_id);
1507 return err;
1508 }
1509
1510 err = gv100_nvlink_rxcal_en(g, BIT(link_id));
1511 if (err) {
1512 nvgpu_err(g, "Failed to RXcal on link: %u", link_id);
1513 return err;
1514 }
1515
1516 err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true);
1517 if (err) {
1518 nvgpu_err(g, "Failed to set data ready link:%u",
1519 link_id);
1520 return err;
1521 }
1522
1523 g->nvlink.enabled_links |= BIT(link_id);
1524 }
1525
1526 nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x",
1527 g->nvlink.enabled_links);
1528
1529 if (g->nvlink.enabled_links)
1530 return 0;
1531
1532 nvgpu_err(g, " No links were enabled");
1533 return -EINVAL;
1534}
1535
1536static int gv100_nvlink_enable_links_post_top(struct gk20a *g, u32 links)
1537{
1538 u32 link_id;
1539 unsigned long enabled_links = (links & g->nvlink.enabled_links) &
1540 ~g->nvlink.initialized_links;
1541 u32 reg;
1542
1543 for_each_set_bit(link_id, &enabled_links, 32) {
1544
1545 /* WAR for HW bug 1888034 */
1546 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r());
1547 reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(),
1548 nvl_sl0_safe_ctrl2_tx_ctr_init_init_f());
1549 reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(),
1550 nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f());
1551 DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg);
1552
1553 gv100_nvlink_initialize_tlc(g, link_id);
1554 gv100_nvlink_initialize_nvlipt(g, link_id);
1555 gv100_nvlink_enable_link_intr(g, link_id, true);
1556
1557 g->nvlink.initialized_links |= BIT(link_id);
1558 };
1559
1560 return 0;
1561}
1562
1563static u32 gv100_nvlink_prbs_gen_en(struct gk20a *g, unsigned long mask)
1564{
1565 u32 reg;
1566 u32 link_id;
1567
1568 for_each_set_bit(link_id, &mask, 32) {
1569 /* Write is required as part of HW sequence */
1570 DLPL_REG_WR32(g, link_id, nvl_sl1_rxslsm_timeout_2_r(), 0);
1571
1572 reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r());
1573 reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(),
1574 nvl_txiobist_config_dpg_prbsseedld_f(0x1));
1575 DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg);
1576
1577 reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r());
1578 reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(),
1579 nvl_txiobist_config_dpg_prbsseedld_f(0x0));
1580 DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg);
1581 }
1582
1583 return 0;
1584}
1585
1586static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask)
1587{
1588 u32 link_id;
1589 struct nvgpu_timeout timeout;
1590 u32 reg;
1591
1592 for_each_set_bit(link_id, &mask, 32) {
1593 /* Timeout from HW specs */
1594 nvgpu_timeout_init(g, &timeout,
1595 8*NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
1596 reg = DLPL_REG_RD32(g, link_id, nvl_br0_cfg_cal_r());
1597 reg = set_field(reg, nvl_br0_cfg_cal_rxcal_m(),
1598 nvl_br0_cfg_cal_rxcal_on_f());
1599 DLPL_REG_WR32(g, link_id, nvl_br0_cfg_cal_r(), reg);
1600
1601 do {
1602 reg = DLPL_REG_RD32(g, link_id,
1603 nvl_br0_cfg_status_cal_r());
1604
1605 if (nvl_br0_cfg_status_cal_rxcal_done_v(reg) == 1)
1606 break;
1607 nvgpu_udelay(5);
1608 } while(!nvgpu_timeout_expired_msg(&timeout,
1609 "timeout on rxcal"));
1610
1611 if (nvgpu_timeout_peek_expired(&timeout))
1612 return -ETIMEDOUT;
1613 }
1614
1615 return 0;
1616}
1617
1618/*
1619 *******************************************************************************
1620 * Internal "ops" functions *
1621 *******************************************************************************
1622 */
1623
1624
1625/*
1626 * Main Nvlink init function. Calls into the Nvlink core API
1627 */
1628int gv100_nvlink_init(struct gk20a *g)
1629{
1630 int err = 0;
1631
1632 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK))
1633 return -ENODEV;
1634
1635 err = nvgpu_nvlink_enumerate(g);
1636 if (err)
1637 return err;
1638
1639 /* Set HSHUB and SG_PHY */
1640 __nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true);
1641 return err;
1642}
1643
1644/*
1645 * Query internal device topology and discover devices in nvlink local
1646 * infrastructure. Initialize register base and offsets
1647 */
1648int gv100_nvlink_discover_link(struct gk20a *g)
1649{
1650 u32 i;
1651 u32 ioctrl_entry_addr;
1652 u8 ioctrl_device_type;
1653 u32 table_entry;
1654 u32 ioctrl_info_entry_type;
1655 u8 ioctrl_discovery_size;
1656 bool is_chain = false;
1657 u8 nvlink_num_devices = 0;
1658 unsigned long available_links = 0;
1659 struct nvgpu_nvlink_device_list *device_table;
1660 u32 err = 0;
1661
1662 /*
1663 * Process Entry 0 & 1 of IOCTRL table to find table size
1664 */
1665 if (g->nvlink.ioctrl_table && g->nvlink.ioctrl_table[0].pri_base_addr) {
1666 ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr;
1667 table_entry = gk20a_readl(g, ioctrl_entry_addr);
1668 ioctrl_info_entry_type = nvlinkip_discovery_common_device_v(table_entry);
1669 } else {
1670 nvgpu_err(g, " Bad IOCTRL PRI Base addr");
1671 return -EINVAL;
1672 }
1673
1674 if (ioctrl_info_entry_type == NVL_DEVICE(ioctrl)) {
1675 ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr + 4;
1676 table_entry = gk20a_readl(g, ioctrl_entry_addr);
1677 ioctrl_discovery_size = nvlinkip_discovery_common_ioctrl_length_v(table_entry);
1678 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL size: %d", ioctrl_discovery_size);
1679 } else {
1680 nvgpu_err(g, " First entry of IOCTRL_DISCOVERY invalid");
1681 return -EINVAL;
1682 }
1683
1684 device_table = nvgpu_kzalloc(g, ioctrl_discovery_size *
1685 sizeof(struct nvgpu_nvlink_device_list));
1686 if (!device_table) {
1687 nvgpu_err(g, " Unable to allocate nvlink device table");
1688 return -ENOMEM;
1689 }
1690
1691 for (i = 0; i < ioctrl_discovery_size; i++) {
1692 ioctrl_entry_addr =
1693 g->nvlink.ioctrl_table[0].pri_base_addr + 4*i;
1694 table_entry = gk20a_readl(g, ioctrl_entry_addr);
1695
1696 nvgpu_log(g, gpu_dbg_nvlink, "parsing ioctrl %d: 0x%08x", i, table_entry);
1697
1698 ioctrl_info_entry_type = nvlinkip_discovery_common_entry_v(table_entry);
1699
1700 if (ioctrl_info_entry_type ==
1701 nvlinkip_discovery_common_entry_invalid_v())
1702 continue;
1703
1704 if (ioctrl_info_entry_type ==
1705 nvlinkip_discovery_common_entry_enum_v()) {
1706
1707 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is ENUM", i);
1708
1709 ioctrl_device_type =
1710 nvlinkip_discovery_common_device_v(table_entry);
1711
1712 if (nvlinkip_discovery_common_chain_v(table_entry) !=
1713 nvlinkip_discovery_common_chain_enable_v()) {
1714
1715 nvgpu_log(g, gpu_dbg_nvlink,
1716 "IOCTRL entry %d is ENUM but no chain",
1717 i);
1718 err = -EINVAL;
1719 break;
1720 }
1721
1722 is_chain = true;
1723 device_table[nvlink_num_devices].valid = true;
1724 device_table[nvlink_num_devices].device_type =
1725 ioctrl_device_type;
1726 device_table[nvlink_num_devices].device_id =
1727 nvlinkip_discovery_common_id_v(table_entry);
1728 device_table[nvlink_num_devices].device_version =
1729 nvlinkip_discovery_common_version_v(
1730 table_entry);
1731 continue;
1732 }
1733
1734 if (ioctrl_info_entry_type ==
1735 nvlinkip_discovery_common_entry_data1_v()) {
1736 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA1", i);
1737
1738 if (is_chain) {
1739 device_table[nvlink_num_devices].pri_base_addr =
1740 nvlinkip_discovery_common_pri_base_v(
1741 table_entry) << 12;
1742
1743 device_table[nvlink_num_devices].intr_enum =
1744 nvlinkip_discovery_common_intr_v(
1745 table_entry);
1746
1747 device_table[nvlink_num_devices].reset_enum =
1748 nvlinkip_discovery_common_reset_v(
1749 table_entry);
1750
1751 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d type = %d base: 0x%08x intr: %d reset: %d",
1752 i,
1753 device_table[nvlink_num_devices].device_type,
1754 device_table[nvlink_num_devices].pri_base_addr,
1755 device_table[nvlink_num_devices].intr_enum,
1756 device_table[nvlink_num_devices].reset_enum);
1757
1758 if (device_table[nvlink_num_devices].device_type ==
1759 NVL_DEVICE(dlpl)) {
1760 device_table[nvlink_num_devices].num_tx =
1761 nvlinkip_discovery_common_dlpl_num_tx_v(table_entry);
1762 device_table[nvlink_num_devices].num_rx =
1763 nvlinkip_discovery_common_dlpl_num_rx_v(table_entry);
1764
1765 nvgpu_log(g, gpu_dbg_nvlink, "DLPL tx: %d rx: %d",
1766 device_table[nvlink_num_devices].num_tx,
1767 device_table[nvlink_num_devices].num_rx);
1768 }
1769
1770 if (nvlinkip_discovery_common_chain_v(table_entry) !=
1771 nvlinkip_discovery_common_chain_enable_v()) {
1772
1773 is_chain = false;
1774 nvlink_num_devices++;
1775 }
1776 }
1777 continue;
1778 }
1779
1780 if (ioctrl_info_entry_type ==
1781 nvlinkip_discovery_common_entry_data2_v()) {
1782
1783 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA2", i);
1784
1785 if (is_chain) {
1786 if (nvlinkip_discovery_common_dlpl_data2_type_v(table_entry)) {
1787 device_table[nvlink_num_devices].pll_master =
1788 nvlinkip_discovery_common_dlpl_data2_master_v(table_entry);
1789 device_table[nvlink_num_devices].pll_master_id =
1790 nvlinkip_discovery_common_dlpl_data2_masterid_v(table_entry);
1791 nvgpu_log(g, gpu_dbg_nvlink, "PLL info: Master: %d, Master ID: %d",
1792 device_table[nvlink_num_devices].pll_master,
1793 device_table[nvlink_num_devices].pll_master_id);
1794 }
1795
1796 if (nvlinkip_discovery_common_chain_v(table_entry) !=
1797 nvlinkip_discovery_common_chain_enable_v()) {
1798
1799 is_chain = false;
1800 nvlink_num_devices++;
1801 }
1802 }
1803 continue;
1804 }
1805 }
1806
1807 g->nvlink.device_table = device_table;
1808 g->nvlink.num_devices = nvlink_num_devices;
1809
1810 /*
1811 * Print table
1812 */
1813 for (i = 0; i < nvlink_num_devices; i++) {
1814 if (device_table[i].valid) {
1815 nvgpu_log(g, gpu_dbg_nvlink, "Device %d - %s", i,
1816 __gv100_device_type_to_str(
1817 device_table[i].device_type));
1818 nvgpu_log(g, gpu_dbg_nvlink, "+Link/Device Id: %d", device_table[i].device_id);
1819 nvgpu_log(g, gpu_dbg_nvlink, "+Version: %d", device_table[i].device_version);
1820 nvgpu_log(g, gpu_dbg_nvlink, "+Base Addr: 0x%08x", device_table[i].pri_base_addr);
1821 nvgpu_log(g, gpu_dbg_nvlink, "+Intr Enum: %d", device_table[i].intr_enum);
1822 nvgpu_log(g, gpu_dbg_nvlink, "+Reset Enum: %d", device_table[i].reset_enum);
1823 if ((device_table[i].device_type == NVL_DEVICE(dlpl)) ||
1824 (device_table[i].device_type == NVL_DEVICE(nvlink))) {
1825 nvgpu_log(g, gpu_dbg_nvlink, "+TX: %d", device_table[i].num_tx);
1826 nvgpu_log(g, gpu_dbg_nvlink, "+RX: %d", device_table[i].num_rx);
1827 nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master: %d", device_table[i].pll_master);
1828 nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master ID: %d", device_table[i].pll_master_id);
1829 }
1830 }
1831 }
1832
1833 for (i = 0; i < nvlink_num_devices; i++) {
1834 if (device_table[i].valid) {
1835
1836 if (device_table[i].device_type == NVL_DEVICE(ioctrl)) {
1837
1838 g->nvlink.ioctrl_type =
1839 device_table[i].device_type;
1840 g->nvlink.ioctrl_base =
1841 device_table[i].pri_base_addr;
1842 continue;
1843 }
1844
1845 if (device_table[i].device_type == NVL_DEVICE(dlpl)) {
1846
1847 g->nvlink.dlpl_type =
1848 device_table[i].device_type;
1849 g->nvlink.dlpl_base[device_table[i].device_id] =
1850 device_table[i].pri_base_addr;
1851 g->nvlink.links[device_table[i].device_id].valid = true;
1852 g->nvlink.links[device_table[i].device_id].g = g;
1853 g->nvlink.links[device_table[i].device_id].dlpl_version =
1854 device_table[i].device_version;
1855 g->nvlink.links[device_table[i].device_id].dlpl_base =
1856 device_table[i].pri_base_addr;
1857 g->nvlink.links[device_table[i].device_id].intr_enum =
1858 device_table[i].intr_enum;
1859 g->nvlink.links[device_table[i].device_id].reset_enum =
1860 device_table[i].reset_enum;
1861 g->nvlink.links[device_table[i].device_id].link_id =
1862 device_table[i].device_id;
1863
1864 /* initiate the PLL master and slave link id to max */
1865 g->nvlink.links[device_table[i].device_id].pll_master_link_id =
1866 NVLINK_MAX_LINKS_SW;
1867 g->nvlink.links[device_table[i].device_id].pll_slave_link_id =
1868 NVLINK_MAX_LINKS_SW;
1869
1870 /* Update Pll master */
1871 if (device_table[i].pll_master)
1872 g->nvlink.links[device_table[i].device_id].pll_master_link_id =
1873 g->nvlink.links[device_table[i].device_id].link_id;
1874 else {
1875 g->nvlink.links[device_table[i].device_id].pll_master_link_id =
1876 device_table[i].pll_master_id;
1877 g->nvlink.links[device_table[i].device_id].pll_slave_link_id =
1878 g->nvlink.links[device_table[i].device_id].link_id;
1879 g->nvlink.links[device_table[i].pll_master_id].pll_slave_link_id =
1880 g->nvlink.links[device_table[i].device_id].link_id;
1881 }
1882
1883 available_links |= BIT(device_table[i].device_id);
1884 continue;
1885 }
1886
1887 if (device_table[i].device_type == NVL_DEVICE(nvltlc)) {
1888
1889 g->nvlink.tl_type = device_table[i].device_type;
1890 g->nvlink.tl_base[device_table[i].device_id] =
1891 device_table[i].pri_base_addr;
1892 g->nvlink.links[device_table[i].device_id].tl_base =
1893 device_table[i].pri_base_addr;
1894 g->nvlink.links[device_table[i].device_id].tl_version =
1895 device_table[i].device_version;
1896 continue;
1897 }
1898
1899 if (device_table[i].device_type == NVL_DEVICE(nvltlc)) {
1900
1901 g->nvlink.tl_type = device_table[i].device_type;
1902 g->nvlink.tl_base[device_table[i].device_id] =
1903 device_table[i].pri_base_addr;
1904 g->nvlink.links[device_table[i].device_id].tl_base =
1905 device_table[i].pri_base_addr;
1906 g->nvlink.links[device_table[i].device_id].tl_version =
1907 device_table[i].device_version;
1908 continue;
1909 }
1910
1911 if (device_table[i].device_type == NVL_DEVICE(ioctrlmif)) {
1912
1913 g->nvlink.mif_type = device_table[i].device_type;
1914 g->nvlink.mif_base[device_table[i].device_id] =
1915 device_table[i].pri_base_addr;
1916 g->nvlink.links[device_table[i].device_id].mif_base =
1917 device_table[i].pri_base_addr;
1918 g->nvlink.links[device_table[i].device_id].mif_version =
1919 device_table[i].device_version;
1920 continue;
1921 }
1922
1923 if (device_table[i].device_type == NVL_DEVICE(nvlipt)) {
1924
1925 g->nvlink.ipt_type =
1926 device_table[i].device_type;
1927 g->nvlink.ipt_base =
1928 device_table[i].pri_base_addr;
1929 g->nvlink.ipt_version =
1930 device_table[i].device_version;
1931 continue;
1932 }
1933
1934 if (device_table[i].device_type == NVL_DEVICE(minion)) {
1935
1936 g->nvlink.minion_type =
1937 device_table[i].device_type;
1938 g->nvlink.minion_base =
1939 device_table[i].pri_base_addr;
1940 g->nvlink.minion_version =
1941 device_table[i].device_version;
1942 continue;
1943 }
1944
1945 if (device_table[i].device_type == NVL_DEVICE(dlpl_multicast)) {
1946
1947 g->nvlink.dlpl_multicast_type =
1948 device_table[i].device_type;
1949 g->nvlink.dlpl_multicast_base =
1950 device_table[i].pri_base_addr;
1951 g->nvlink.dlpl_multicast_version =
1952 device_table[i].device_version;
1953 continue;
1954 }
1955 if (device_table[i].device_type == NVL_DEVICE(nvltlc_multicast)) {
1956
1957 g->nvlink.tl_multicast_type =
1958 device_table[i].device_type;
1959 g->nvlink.tl_multicast_base =
1960 device_table[i].pri_base_addr;
1961 g->nvlink.tl_multicast_version =
1962 device_table[i].device_version;
1963 continue;
1964 }
1965
1966 if (device_table[i].device_type == NVL_DEVICE(ioctrlmif_multicast)) {
1967
1968 g->nvlink.mif_multicast_type =
1969 device_table[i].device_type;
1970 g->nvlink.mif_multicast_base =
1971 device_table[i].pri_base_addr;
1972 g->nvlink.mif_multicast_version =
1973 device_table[i].device_version;
1974 continue;
1975 }
1976
1977 }
1978 }
1979
1980 g->nvlink.discovered_links = (u32) available_links;
1981
1982 nvgpu_log(g, gpu_dbg_nvlink, "Nvlink Tree:");
1983 nvgpu_log(g, gpu_dbg_nvlink, "+ Available Links: 0x%08lx", available_links);
1984 nvgpu_log(g, gpu_dbg_nvlink, "+ Per-Link Devices:");
1985
1986 for_each_set_bit(i, &available_links, 32) {
1987 nvgpu_log(g, gpu_dbg_nvlink, "-- Link %d Dl/Pl Base: 0x%08x TLC Base: 0x%08x MIF Base: 0x%08x",
1988 i, g->nvlink.dlpl_base[i], g->nvlink.tl_base[i], g->nvlink.mif_base[i]);
1989 }
1990
1991 nvgpu_log(g, gpu_dbg_nvlink, "+ IOCTRL Base: 0x%08x", g->nvlink.ioctrl_base);
1992 nvgpu_log(g, gpu_dbg_nvlink, "+ NVLIPT Base: 0x%08x", g->nvlink.ipt_base);
1993 nvgpu_log(g, gpu_dbg_nvlink, "+ MINION Base: 0x%08x", g->nvlink.minion_base);
1994 nvgpu_log(g, gpu_dbg_nvlink, "+ DLPL MCAST Base: 0x%08x", g->nvlink.dlpl_multicast_base);
1995 nvgpu_log(g, gpu_dbg_nvlink, "+ TLC MCAST Base: 0x%08x", g->nvlink.tl_multicast_base);
1996 nvgpu_log(g, gpu_dbg_nvlink, "+ MIF MCAST Base: 0x%08x", g->nvlink.mif_multicast_base);
1997
1998 if (!g->nvlink.minion_version) {
1999 nvgpu_err(g, "Unsupported MINION version");
2000
2001 nvgpu_kfree(g, device_table);
2002 g->nvlink.device_table = NULL;
2003 g->nvlink.num_devices = 0;
2004 return -EINVAL;
2005 }
2006
2007 return err;
2008}
2009
2010/*
2011 * Query IOCTRL for device discovery
2012 */
2013int gv100_nvlink_discover_ioctrl(struct gk20a *g)
2014{
2015 u32 i;
2016 struct nvgpu_nvlink_ioctrl_list *ioctrl_table;
2017 u32 table_entry;
2018 u32 devinfo_type;
2019 bool is_ioctrl = false;
2020 bool is_chain = false;
2021 u32 io_num_entries = 0;
2022
2023 ioctrl_table = nvgpu_kzalloc(g, top_device_info__size_1_v() *
2024 sizeof(struct nvgpu_nvlink_ioctrl_list));
2025
2026 if (!ioctrl_table) {
2027 nvgpu_err(g, "failed to allocate memory for nvlink io table");
2028 return -ENOMEM;
2029 }
2030
2031 for (i = 0; i < top_device_info__size_1_v(); i++) {
2032 table_entry = gk20a_readl(g, top_device_info_r(i));
2033
2034 devinfo_type = top_device_info_entry_v(table_entry);
2035
2036 if (devinfo_type == top_device_info_entry_not_valid_v())
2037 continue;
2038
2039 if (devinfo_type == top_device_info_entry_engine_type_v()) {
2040 if (top_device_info_type_enum_v(table_entry) ==
2041 top_device_info_type_enum_ioctrl_v())
2042 is_ioctrl = true;
2043 else {
2044 is_ioctrl = false;
2045 continue;
2046 }
2047
2048 if (top_device_info_chain_v(table_entry) !=
2049 top_device_info_chain_enable_v())
2050 break;
2051
2052 is_chain = true;
2053 ioctrl_table[io_num_entries].valid = true;
2054 continue;
2055 }
2056
2057 if (devinfo_type == top_device_info_entry_data_v()) {
2058 if (is_ioctrl && is_chain) {
2059
2060 ioctrl_table[io_num_entries].pri_base_addr =
2061 top_device_info_data_pri_base_v(table_entry) <<
2062 top_device_info_data_pri_base_align_v();
2063
2064 if (top_device_info_chain_v(table_entry) !=
2065 top_device_info_chain_enable_v()) {
2066 is_chain = false;
2067 io_num_entries++;
2068 }
2069 }
2070 continue;
2071 }
2072
2073 if (devinfo_type == top_device_info_entry_enum_v()) {
2074 if (is_ioctrl && is_chain) {
2075
2076 ioctrl_table[io_num_entries].intr_enum =
2077 top_device_info_intr_v(table_entry);
2078 ioctrl_table[io_num_entries].reset_enum =
2079 top_device_info_reset_v(table_entry);
2080
2081 if (top_device_info_chain_v(table_entry) !=
2082 top_device_info_chain_enable_v()) {
2083 is_chain = false;
2084 io_num_entries++;
2085 }
2086 }
2087 continue;
2088 }
2089 }
2090
2091 if (io_num_entries == 0 || !ioctrl_table[0].pri_base_addr) {
2092 nvgpu_err(g, "No NVLINK io found");
2093 nvgpu_kfree(g, ioctrl_table);
2094 return -EINVAL;
2095 }
2096
2097 g->nvlink.ioctrl_table = ioctrl_table;
2098 g->nvlink.io_num_entries = io_num_entries;
2099
2100
2101 for (i =0; i < io_num_entries; i++)
2102 nvgpu_log(g, gpu_dbg_nvlink,
2103 "Device %d : Pri Base Addr = 0x%0x Intr = %d Reset = %d",
2104 i, ioctrl_table[i].pri_base_addr, ioctrl_table[i].intr_enum,
2105 ioctrl_table[i].reset_enum);
2106
2107 return 0;
2108}
2109
2110/*
2111 *******************************************************************************
2112 * NVLINK API FUNCTIONS *
2113 *******************************************************************************
2114 */
2115
2116/*
2117 * Performs link level initialization like phy inits, AN0 and interrupts
2118 */
2119
2120int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask)
2121{
2122 int err;
2123
2124 err = gv100_nvlink_enable_links_pre_top(g, mask);
2125 if (err) {
2126 nvgpu_err(g, "Pre topology failed for links %lx", mask);
2127 return err;
2128 }
2129
2130 nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx",
2131 mask & g->nvlink.enabled_links);
2132 err = gv100_nvlink_enable_links_post_top(g, mask);
2133
2134 return err;
2135}
2136
2137/*
2138 * Performs memory interface initialization
2139 */
2140
2141int gv100_nvlink_interface_init(struct gk20a *g)
2142{
2143 unsigned long mask = g->nvlink.enabled_links;
2144 u32 link_id;
2145
2146 for_each_set_bit(link_id, &mask, 32) {
2147 gv100_nvlink_initialize_mif(g, link_id);
2148 gv100_nvlink_mif_intr_enable(g, link_id, true);
2149 }
2150
2151 return 0;
2152}
2153
2154int gv100_nvlink_reg_init(struct gk20a *g)
2155{
2156 u32 i = 0;
2157 u32 count = 0;
2158 struct __nvlink_reginit *reg;
2159 enum nvgpu_nvlink_endp endp;
2160 int err;
2161 u32 link_id;
2162 unsigned long mask = g->nvlink.enabled_links;
2163 struct nvgpu_nvlink_link *link;
2164
2165 /* Apply automated reg init flow for PROD settings */
2166 for_each_set_bit(link_id, &mask, 32) {
2167
2168 link = &g->nvlink.links[link_id];
2169 if (!link->remote_info.is_connected)
2170 continue;
2171
2172 endp = link->remote_info.device_type;
2173 err = gv100_nvlink_get_tlc_reginit(endp, &reg, &count);
2174 if (err) {
2175 nvgpu_err(g, "no reginit for endp=%u", endp);
2176 continue;
2177 }
2178
2179 for (i = 0; i < count; i++) {
2180 TLC_REG_WR32(g, link_id, reg->addr, reg->value);
2181 reg++;
2182 }
2183 }
2184 return 0;
2185}
2186
2187/*
2188 * Shutdown device. This should tear down Nvlink connection.
2189 * For now return.
2190 */
2191int gv100_nvlink_shutdown(struct gk20a *g)
2192{
2193 return 0;
2194}
2195
2196/*
2197 * Get link state
2198 */
2199u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id)
2200{
2201 return DLPL_REG_RD32(g, link_id, nvl_link_state_r()) &
2202 nvl_link_state_state_m();
2203}
2204
2205/* Get link mode */
2206u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id)
2207{
2208 u32 state;
2209 if (!(BIT(link_id) & g->nvlink.discovered_links))
2210 return nvgpu_nvlink_link__last;
2211
2212 state = nvl_link_state_state_v(
2213 g->ops.nvlink.link_get_state(g, link_id));
2214
2215 if (state == nvl_link_state_state_init_v())
2216 return nvgpu_nvlink_link_off;
2217 if (state == nvl_link_state_state_hwcfg_v())
2218 return nvgpu_nvlink_link_detect;
2219 if (state == nvl_link_state_state_swcfg_v())
2220 return nvgpu_nvlink_link_safe;
2221 if (state == nvl_link_state_state_active_v())
2222 return nvgpu_nvlink_link_hs;
2223 if (state == nvl_link_state_state_fault_v())
2224 return nvgpu_nvlink_link_fault;
2225 if ((state == nvl_link_state_state_rcvy_ac_v()) ||
2226 (state == nvl_link_state_state_rcvy_sw_v()) ||
2227 (state == nvl_link_state_state_rcvy_rx_v()))
2228 return nvgpu_nvlink_link_recovery;
2229
2230 return nvgpu_nvlink_link_off;
2231}
2232
2233/* Set Link mode */
2234int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode)
2235{
2236 u32 state;
2237 u32 reg;
2238 u32 err = 0;
2239
2240 nvgpu_log(g, gpu_dbg_nvlink, "link :%d, mode:%u", link_id, mode);
2241
2242 if (!(BIT(link_id) & g->nvlink.enabled_links))
2243 return -EINVAL;
2244
2245 state = nvl_link_state_state_v(
2246 g->ops.nvlink.link_get_state(g, link_id));
2247
2248 switch (mode) {
2249 case nvgpu_nvlink_link_safe:
2250 if (state == nvl_link_state_state_swcfg_v()) {
2251 nvgpu_warn(g, "link is already in safe mode");
2252 break;
2253 }
2254 if (state == nvl_link_state_state_hwcfg_v()) {
2255 nvgpu_warn(g, "link is transitioning to safe mode");
2256 break;
2257 }
2258
2259 if (state == nvl_link_state_state_init_v()) {
2260 /* Off to Safe transition */
2261 reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
2262 reg = set_field(reg, nvl_link_change_newstate_m(),
2263 nvl_link_change_newstate_hwcfg_f());
2264 reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
2265 nvl_link_change_oldstate_mask_dontcare_f());
2266 reg = set_field(reg, nvl_link_change_action_m(),
2267 nvl_link_change_action_ltssm_change_f());
2268 DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
2269 } else if (state == nvl_link_state_state_active_v()) {
2270 /* TODO:
2271 * Disable PM first since we are moving out active
2272 * state
2273 */
2274 reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
2275 reg = set_field(reg, nvl_link_change_newstate_m(),
2276 nvl_link_change_newstate_swcfg_f());
2277 reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
2278 nvl_link_change_oldstate_mask_dontcare_f());
2279 reg = set_field(reg, nvl_link_change_action_m(),
2280 nvl_link_change_action_ltssm_change_f());
2281 DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
2282 }
2283 break;
2284
2285 case nvgpu_nvlink_link_hs:
2286 if (state == nvl_link_state_state_active_v()) {
2287 nvgpu_err(g, "link is already in active mode");
2288 break;
2289 }
2290 if (state == nvl_link_state_state_init_v()) {
2291 nvgpu_err(g, "link cannot be taken from init state");
2292 return -EPERM;
2293 }
2294
2295 reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
2296 reg = set_field(reg, nvl_link_change_newstate_m(),
2297 nvl_link_change_newstate_active_f());
2298 reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
2299 nvl_link_change_oldstate_mask_dontcare_f());
2300 reg = set_field(reg, nvl_link_change_action_m(),
2301 nvl_link_change_action_ltssm_change_f());
2302 DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
2303 break;
2304
2305 case nvgpu_nvlink_link_off:
2306 if (state == nvl_link_state_state_active_v()) {
2307 nvgpu_err(g, "link cannot be taken from active to init");
2308 return -EPERM;
2309 }
2310 if (state == nvl_link_state_state_init_v()) {
2311 nvgpu_err(g, "link already in init state");
2312 }
2313
2314 /* GV100 UPHY is handled by MINION */
2315 break;
2316 /* 1/8 th mode not supported */
2317 case nvgpu_nvlink_link_enable_pm:
2318 case nvgpu_nvlink_link_disable_pm:
2319 return -EPERM;
2320 case nvgpu_nvlink_link_disable_err_detect:
2321 /* Disable Link interrupts */
2322 gv100_nvlink_dlpl_intr_enable(g, link_id, false);
2323 break;
2324 case nvgpu_nvlink_link_lane_disable:
2325 err = gv100_nvlink_minion_lane_disable(g, link_id, true);
2326 break;
2327 case nvgpu_nvlink_link_lane_shutdown:
2328 err = gv100_nvlink_minion_lane_shutdown(g, link_id, true);
2329 break;
2330 default:
2331 nvgpu_err(g, "Unhandled mode %x", mode);
2332 break;
2333 }
2334
2335 return err;
2336}
2337
2338static u32 gv100_nvlink_link_sublink_check_change(struct gk20a *g, u32 link_id)
2339{
2340 struct nvgpu_timeout timeout;
2341 u32 reg;
2342
2343 nvgpu_timeout_init(g, &timeout,
2344 NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
2345 /* Poll for sublink status */
2346 do {
2347 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2348
2349 if (nvl_sublink_change_status_v(reg) ==
2350 nvl_sublink_change_status_done_v())
2351 break;
2352 if (nvl_sublink_change_status_v(reg) ==
2353 nvl_sublink_change_status_fault_v()) {
2354 nvgpu_err(g, "Fault detected in sublink change");
2355 return -EFAULT;
2356 }
2357 nvgpu_udelay(5);
2358 } while(!nvgpu_timeout_expired_msg(&timeout, "timeout on sublink rdy"));
2359
2360 if (nvgpu_timeout_peek_expired(&timeout))
2361 return -ETIMEDOUT;
2362 return-0;
2363}
2364
2365int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id,
2366 bool is_rx_sublink, u32 mode)
2367{
2368 int err = 0;
2369 u32 rx_sublink_state, tx_sublink_state;
2370 u32 reg;
2371
2372 if (!(BIT(link_id) & g->nvlink.enabled_links))
2373 return -EINVAL;
2374
2375 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2376 if (err)
2377 return err;
2378
2379 rx_sublink_state = g->ops.nvlink.get_rx_sublink_state(g, link_id);
2380 tx_sublink_state = g->ops.nvlink.get_tx_sublink_state(g, link_id);
2381
2382 switch (mode) {
2383 case nvgpu_nvlink_sublink_tx_hs:
2384 if (tx_sublink_state ==
2385 nvl_sl0_slsm_status_tx_primary_state_hs_v()) {
2386 nvgpu_err(g, " TX already in HS");
2387 break;
2388 } else if (tx_sublink_state ==
2389 nvl_sl0_slsm_status_tx_primary_state_off_v()) {
2390 nvgpu_err(g, "TX cannot be do from OFF to HS");
2391 return -EPERM;
2392 }
2393
2394 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2395 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2396 nvl_sublink_change_newstate_hs_f());
2397 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2398 nvl_sublink_change_sublink_tx_f());
2399 reg = set_field(reg, nvl_sublink_change_action_m(),
2400 nvl_sublink_change_action_slsm_change_f());
2401 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2402
2403 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2404 if (err) {
2405 nvgpu_err(g, "Error in TX to HS");
2406 return err;
2407 }
2408 break;
2409 case nvgpu_nvlink_sublink_tx_common:
2410 err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true);
2411 break;
2412 case nvgpu_nvlink_sublink_tx_common_disable:
2413 /* NOP */
2414 break;
2415 case nvgpu_nvlink_sublink_tx_data_ready:
2416 err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true);
2417 break;
2418 case nvgpu_nvlink_sublink_tx_prbs_en:
2419 err = gv100_nvlink_prbs_gen_en(g, BIT(link_id));
2420 break;
2421 case nvgpu_nvlink_sublink_tx_safe:
2422 if (tx_sublink_state ==
2423 nvl_sl0_slsm_status_tx_primary_state_safe_v()) {
2424 nvgpu_err(g, "TX already SAFE: %d", link_id);
2425 break;
2426 }
2427
2428 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2429 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2430 nvl_sublink_change_newstate_safe_f());
2431 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2432 nvl_sublink_change_sublink_tx_f());
2433 reg = set_field(reg, nvl_sublink_change_action_m(),
2434 nvl_sublink_change_action_slsm_change_f());
2435 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2436
2437 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2438 if (err) {
2439 nvgpu_err(g, "Error in TX to SAFE");
2440 return err;
2441 }
2442 break;
2443 case nvgpu_nvlink_sublink_tx_off:
2444 if (tx_sublink_state ==
2445 nvl_sl0_slsm_status_tx_primary_state_off_v()) {
2446 nvgpu_err(g, "TX already OFF: %d", link_id);
2447 break;
2448 } else if (tx_sublink_state ==
2449 nvl_sl0_slsm_status_tx_primary_state_hs_v()) {
2450 nvgpu_err(g, " TX cannot go off from HS %d", link_id);
2451 return -EPERM;
2452 }
2453
2454 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2455 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2456 nvl_sublink_change_newstate_off_f());
2457 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2458 nvl_sublink_change_sublink_tx_f());
2459 reg = set_field(reg, nvl_sublink_change_action_m(),
2460 nvl_sublink_change_action_slsm_change_f());
2461 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2462
2463 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2464 if (err) {
2465 nvgpu_err(g, "Error in TX to OFF");
2466 return err;
2467 }
2468 break;
2469
2470 /* RX modes */
2471 case nvgpu_nvlink_sublink_rx_hs:
2472 case nvgpu_nvlink_sublink_rx_safe:
2473 break;
2474 case nvgpu_nvlink_sublink_rx_off:
2475 if (rx_sublink_state ==
2476 nvl_sl1_slsm_status_rx_primary_state_off_v()) {
2477 nvgpu_err(g, "RX already OFF: %d", link_id);
2478 break;
2479 } else if (rx_sublink_state ==
2480 nvl_sl1_slsm_status_rx_primary_state_hs_v()) {
2481 nvgpu_err(g, " RX cannot go off from HS %d", link_id);
2482 return -EPERM;
2483 }
2484
2485 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2486 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2487 nvl_sublink_change_newstate_off_f());
2488 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2489 nvl_sublink_change_sublink_rx_f());
2490 reg = set_field(reg, nvl_sublink_change_action_m(),
2491 nvl_sublink_change_action_slsm_change_f());
2492 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2493
2494 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2495 if (err) {
2496 nvgpu_err(g, "Error in RX to OFF");
2497 return err;
2498 }
2499 break;
2500 case nvgpu_nvlink_sublink_rx_rxcal:
2501 err = gv100_nvlink_rxcal_en(g, BIT(link_id));
2502 break;
2503
2504 default:
2505 if ((is_rx_sublink) && ((mode < nvgpu_nvlink_sublink_rx_hs) ||
2506 (mode >= nvgpu_nvlink_sublink_rx__last))) {
2507 nvgpu_err(g, "Unsupported RX mode %u", mode);
2508 return -EINVAL;
2509 }
2510 if (mode >= nvgpu_nvlink_sublink_tx__last) {
2511 nvgpu_err(g, "Unsupported TX mode %u", mode);
2512 return -EINVAL;
2513 }
2514 nvgpu_err(g, "MODE %u", mode);
2515 }
2516
2517 if (err)
2518 nvgpu_err(g, " failed on set_sublink_mode");
2519 return err;
2520}
2521
2522u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id,
2523 bool is_rx_sublink)
2524{
2525 u32 state;
2526
2527 if (!(BIT(link_id) & g->nvlink.discovered_links)) {
2528 if (!is_rx_sublink)
2529 return nvgpu_nvlink_sublink_tx__last;
2530 return nvgpu_nvlink_sublink_rx__last;
2531 }
2532
2533 if (!is_rx_sublink) {
2534 state = g->ops.nvlink.get_tx_sublink_state(g, link_id);
2535 if (state == nvl_sl0_slsm_status_tx_primary_state_hs_v())
2536 return nvgpu_nvlink_sublink_tx_hs;
2537 if (state == nvl_sl0_slsm_status_tx_primary_state_eighth_v())
2538 return nvgpu_nvlink_sublink_tx_single_lane;
2539 if (state == nvl_sl0_slsm_status_tx_primary_state_safe_v())
2540 return nvgpu_nvlink_sublink_tx_safe;
2541 if (state == nvl_sl0_slsm_status_tx_primary_state_off_v())
2542 return nvgpu_nvlink_sublink_tx_off;
2543 return nvgpu_nvlink_sublink_tx_off;
2544 } else {
2545 state = g->ops.nvlink.get_rx_sublink_state(g, link_id);
2546 if (state == nvl_sl1_slsm_status_rx_primary_state_hs_v())
2547 return nvgpu_nvlink_sublink_rx_hs;
2548 if (state == nvl_sl1_slsm_status_rx_primary_state_eighth_v())
2549 return nvgpu_nvlink_sublink_rx_single_lane;
2550 if (state == nvl_sl1_slsm_status_rx_primary_state_safe_v())
2551 return nvgpu_nvlink_sublink_rx_safe;
2552 if (state == nvl_sl1_slsm_status_rx_primary_state_off_v())
2553 return nvgpu_nvlink_sublink_rx_off;
2554 return nvgpu_nvlink_sublink_rx_off;
2555 }
2556 return nvgpu_nvlink_sublink_tx__last;
2557}
2558
2559/*
2560 * Get TX sublink state
2561 */
2562u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id)
2563{
2564 u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_slsm_status_tx_r());
2565
2566 return nvl_sl0_slsm_status_tx_primary_state_v(reg);
2567}
2568
2569/*
2570 * Get RX sublink state
2571 */
2572u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id)
2573{
2574 u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_slsm_status_rx_r());
2575
2576 return nvl_sl1_slsm_status_rx_primary_state_v(reg);
2577}
2578
2579/*
2580 * Performs nvlink device level initialization by discovering the topology
2581 * taking device out of reset, boot minion, set clocks up and common interrupts
2582 */
2583int gv100_nvlink_early_init(struct gk20a *g)
2584{
2585 int err = 0;
2586
2587 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK))
2588 return -EINVAL;
2589
2590 err = nvgpu_bios_get_nvlink_config_data(g);
2591 if (err) {
2592 nvgpu_err(g, "failed to read nvlink vbios data");
2593 goto nvlink_init_exit;
2594 }
2595
2596 err = g->ops.nvlink.discover_ioctrl(g);
2597 if (err)
2598 goto nvlink_init_exit;
2599
2600 /* Enable NVLINK in MC */
2601 g->ops.mc.reset(g, mc_enable_nvlink_enabled_f());
2602
2603 err = g->ops.nvlink.discover_link(g);
2604 if (err || g->nvlink.discovered_links == 0) {
2605 nvgpu_err(g, "No links available");
2606 goto nvlink_init_exit;
2607 }
2608
2609 nvgpu_flcn_sw_init(g, FALCON_ID_MINION);
2610
2611 g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask;
2612 nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)",
2613 g->nvlink.link_disable_mask);
2614
2615 /* Links in reset should be removed from initialized link sw state */
2616 g->nvlink.initialized_links &= __gv100_nvlink_get_link_reset_mask(g);
2617
2618 nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x (from DT)",
2619 g->nvlink.connected_links);
2620
2621 /* Track unconnected links */
2622 g->nvlink.discovered_links &= g->nvlink.connected_links;
2623
2624 nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)",
2625 g->nvlink.discovered_links);
2626
2627 if (hweight32(g->nvlink.discovered_links) > 1) {
2628 nvgpu_err(g, "more than one link enabled");
2629 err = -EINVAL;
2630 goto nvlink_init_exit;
2631 }
2632
2633 /* For now set default speed */
2634 g->nvlink.speed = nvgpu_nvlink_speed_default;
2635
2636 err = __gv100_nvlink_state_load_hal(g);
2637 if (err) {
2638 nvgpu_err(g, " failed Nvlink state load");
2639 goto nvlink_init_exit;
2640 }
2641 err = gv100_nvlink_minion_configure_ac_coupling(g,
2642 g->nvlink.ac_coupling_mask, true);
2643 if (err) {
2644 nvgpu_err(g, " failed Nvlink state load");
2645 goto nvlink_init_exit;
2646 }
2647
2648 /* Program clocks */
2649 gv100_nvlink_prog_alt_clk(g);
2650
2651nvlink_init_exit:
2652 return err;
2653}
2654
diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.h b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h
new file mode 100644
index 00000000..4ac8b907
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h
@@ -0,0 +1,60 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef NVGPU_NVLINK_GV100_H
24#define NVGPU_NVLINK_GV100_H
25
26struct gk20a;
27
28#define MINION_REG_RD32(g, off) gk20a_readl(g, g->nvlink.minion_base + (off))
29#define MINION_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.minion_base + (off), (v))
30#define IOCTRL_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ioctrl_base + (off))
31#define IOCTRL_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ioctrl_base + (off), (v));
32#define MIF_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].mif_base + (off))
33#define MIF_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].mif_base + (off), (v))
34#define IPT_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ipt_base + (off))
35#define IPT_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ipt_base + (off), (v))
36#define TLC_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].tl_base + (off))
37#define TLC_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].tl_base + (off), (v))
38#define DLPL_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].dlpl_base + (off))
39#define DLPL_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].dlpl_base + (off), (v))
40
41int gv100_nvlink_discover_ioctrl(struct gk20a *g);
42int gv100_nvlink_discover_link(struct gk20a *g);
43int gv100_nvlink_init(struct gk20a *g);
44int gv100_nvlink_isr(struct gk20a *g);
45/* API */
46int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask);
47u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id);
48u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id);
49int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode);
50u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id,
51 bool is_rx_sublink);
52u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id);
53u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id);
54int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id,
55 bool is_rx_sublink, u32 mode);
56int gv100_nvlink_interface_init(struct gk20a *g);
57int gv100_nvlink_reg_init(struct gk20a *g);
58int gv100_nvlink_shutdown(struct gk20a *g);
59int gv100_nvlink_early_init(struct gk20a *g);
60#endif
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
index 89f3f0b9..97ab7aab 100644
--- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
@@ -482,7 +482,7 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id,
482 eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); 482 eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id));
483 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); 483 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat);
484 484
485 if (gv11b_mc_is_stall_and_eng_intr_pending(g, act_eng_id)) { 485 if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id)) {
486 stall_intr = true; 486 stall_intr = true;
487 nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, 487 nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr,
488 "stall intr set, " 488 "stall intr set, "
diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
index 6fdac85a..5282af05 100644
--- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
@@ -658,6 +658,8 @@ static const struct gpu_ops gv11b_ops = {
658 .boot_0 = gk20a_mc_boot_0, 658 .boot_0 = gk20a_mc_boot_0,
659 .is_intr1_pending = mc_gp10b_is_intr1_pending, 659 .is_intr1_pending = mc_gp10b_is_intr1_pending,
660 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, 660 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending,
661 .is_stall_and_eng_intr_pending =
662 gv11b_mc_is_stall_and_eng_intr_pending,
661 }, 663 },
662 .debug = { 664 .debug = {
663 .show_dump = gk20a_debug_show_dump, 665 .show_dump = gk20a_debug_show_dump,
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h
index c59a6e6b..a57ea97a 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h
@@ -892,6 +892,14 @@ static inline u32 minion_nvlink_link_intr_code_dlreq_f(void)
892{ 892{
893 return 0x2U; 893 return 0x2U;
894} 894}
895static inline u32 minion_nvlink_link_intr_code_pmdisabled_v(void)
896{
897 return 0x00000003U;
898}
899static inline u32 minion_nvlink_link_intr_code_pmdisabled_f(void)
900{
901 return 0x3U;
902}
895static inline u32 minion_nvlink_link_intr_subcode_f(u32 v) 903static inline u32 minion_nvlink_link_intr_subcode_f(u32 v)
896{ 904{
897 return (v & 0xffU) << 8U; 905 return (v & 0xffU) << 8U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h
index 48851ff1..18e0aab6 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h
@@ -182,11 +182,10 @@ struct nvgpu_nvlink_dev {
182 u8 train_at_boot; 182 u8 train_at_boot;
183 u32 ac_coupling_mask; 183 u32 ac_coupling_mask;
184 184
185 u32 init_disabled_links;
186 u32 connected_links; 185 u32 connected_links;
187 u32 initialized_links; 186 u32 initialized_links;
188 u32 enabled_links; 187 u32 enabled_links;
189 u32 topology_connected_links; 188 u32 init_pll_done;
190 189
191 enum nvgpu_nvlink_speed speed; 190 enum nvgpu_nvlink_speed speed;
192 191
@@ -195,12 +194,18 @@ struct nvgpu_nvlink_dev {
195 u32 hshub_config1; 194 u32 hshub_config1;
196 u32 hshub_config2; 195 u32 hshub_config2;
197 u32 hshub_config6; 196 u32 hshub_config6;
197
198 /* tlc cached errors */
199 u32 tlc_rx_err_status_0[NVLINK_MAX_LINKS_SW];
200 u32 tlc_rx_err_status_1[NVLINK_MAX_LINKS_SW];
201 u32 tlc_tx_err_status_0[NVLINK_MAX_LINKS_SW];
202
198 /* priv struct */ 203 /* priv struct */
199 void *priv; 204 void *priv;
200}; 205};
201 206
202 207
203u32 nvgpu_nvlink_enumerate(struct gk20a *g); 208int nvgpu_nvlink_enumerate(struct gk20a *g);
204u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); 209int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off);
205u32 nvgpu_nvlink_probe(struct gk20a *g); 210int nvgpu_nvlink_probe(struct gk20a *g);
206#endif 211#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-07 18:21:25 -0400