summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/gpu/nvgpu/Makefile3
-rw-r--r--drivers/gpu/nvgpu/common/linux/nvgpu_mem.c4
-rw-r--r--drivers/gpu/nvgpu/common/linux/nvlink.c46
-rw-r--r--drivers/gpu/nvgpu/gk20a/gk20a.h10
-rw-r--r--drivers/gpu/nvgpu/gv100/flcn_gv100.c55
-rw-r--r--drivers/gpu/nvgpu/gv100/flcn_gv100.h27
-rw-r--r--drivers/gpu/nvgpu/gv100/hal_gv100.c49
-rw-r--r--drivers/gpu/nvgpu/gv100/mc_gv100.c94
-rw-r--r--drivers/gpu/nvgpu/gv100/mc_gv100.h30
-rw-r--r--drivers/gpu/nvgpu/gv100/nvlink_gv100.c2654
-rw-r--r--drivers/gpu/nvgpu/gv100/nvlink_gv100.h60
-rw-r--r--drivers/gpu/nvgpu/gv11b/fifo_gv11b.c2
-rw-r--r--drivers/gpu/nvgpu/gv11b/hal_gv11b.c2
-rw-r--r--drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h8
-rw-r--r--drivers/gpu/nvgpu/include/nvgpu/nvlink.h15
15 files changed, 3016 insertions, 43 deletions
diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile
index 4169bc9b..d9d1da95 100644
--- a/drivers/gpu/nvgpu/Makefile
+++ b/drivers/gpu/nvgpu/Makefile
@@ -246,6 +246,9 @@ nvgpu-y += \
246 gv100/fifo_gv100.o \ 246 gv100/fifo_gv100.o \
247 gv100/gr_gv100.o \ 247 gv100/gr_gv100.o \
248 gv100/regops_gv100.o \ 248 gv100/regops_gv100.o \
249 gv100/flcn_gv100.o \
250 gv100/mc_gv100.o \
251 gv100/nvlink_gv100.o \
249 gv100/hal_gv100.o \ 252 gv100/hal_gv100.o \
250 gv100/pmu_gv100.o \ 253 gv100/pmu_gv100.o \
251 pstate/pstate.o \ 254 pstate/pstate.o \
diff --git a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
index e4b4be06..206b83e1 100644
--- a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
+++ b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. 2 * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
3 * 3 *
4 * This program is free software; you can redistribute it and/or modify it 4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License, 5 * under the terms and conditions of the GNU General Public License,
@@ -536,6 +536,8 @@ static u64 nvgpu_mem_linux_sgl_gpu_addr(struct gk20a *g, void *sgl,
536static bool nvgpu_mem_linux_sgt_iommuable(struct gk20a *g, 536static bool nvgpu_mem_linux_sgt_iommuable(struct gk20a *g,
537 struct nvgpu_sgt *sgt) 537 struct nvgpu_sgt *sgt)
538{ 538{
539 if (nvgpu_is_enabled(g, NVGPU_MM_USE_PHYSICAL_SG))
540 return false;
539 return true; 541 return true;
540} 542}
541 543
diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c
index 6cf74048..61dc8bc7 100644
--- a/drivers/gpu/nvgpu/common/linux/nvlink.c
+++ b/drivers/gpu/nvgpu/common/linux/nvlink.c
@@ -40,7 +40,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev)
40 40
41 /* Lets find the detected link */ 41 /* Lets find the detected link */
42 if (g->nvlink.initialized_links) 42 if (g->nvlink.initialized_links)
43 link_id = fls(g->nvlink.initialized_links); 43 link_id = ffs(g->nvlink.initialized_links) - 1;
44 else 44 else
45 return NVLINK_MAX_LINKS_SW; 45 return NVLINK_MAX_LINKS_SW;
46 46
@@ -49,6 +49,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev)
49 49
50 return NVLINK_MAX_LINKS_SW; 50 return NVLINK_MAX_LINKS_SW;
51} 51}
52
52static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) 53static int nvgpu_nvlink_early_init(struct nvlink_device *ndev)
53{ 54{
54 struct gk20a *g = (struct gk20a *) ndev->priv; 55 struct gk20a *g = (struct gk20a *) ndev->priv;
@@ -75,9 +76,10 @@ static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev)
75 * First check the topology and setup connectivity 76 * First check the topology and setup connectivity
76 * HACK: we are only enabling one link for now!!! 77 * HACK: we are only enabling one link for now!!!
77 */ 78 */
78 link_id = fls(g->nvlink.discovered_links); 79 link_id = ffs(g->nvlink.discovered_links) - 1;
79 g->nvlink.links[link_id].remote_info.is_connected = true; 80 g->nvlink.links[link_id].remote_info.is_connected = true;
80 81 g->nvlink.links[link_id].remote_info.device_type =
82 nvgpu_nvlink_endp_tegra;
81 err = g->ops.nvlink.link_early_init(g, BIT(link_id)); 83 err = g->ops.nvlink.link_early_init(g, BIT(link_id));
82 84
83 if (err == 0) { 85 if (err == 0) {
@@ -262,6 +264,7 @@ static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev,
262 return -EINVAL; 264 return -EINVAL;
263 265
264 mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); 266 mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink);
267
265 switch (mode) { 268 switch (mode) {
266 case nvgpu_nvlink_sublink_tx_hs: 269 case nvgpu_nvlink_sublink_tx_hs:
267 return NVLINK_TX_HS; 270 return NVLINK_TX_HS;
@@ -386,9 +389,9 @@ static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev,
386} 389}
387#endif 390#endif
388 391
389u32 nvgpu_nvlink_enumerate(struct gk20a *g) 392int nvgpu_nvlink_enumerate(struct gk20a *g)
390{ 393{
391 u32 err = -ENODEV; 394 int err = -ENODEV;
392#ifdef CONFIG_TEGRA_NVLINK 395#ifdef CONFIG_TEGRA_NVLINK
393 struct nvlink_device *ndev; 396 struct nvlink_device *ndev;
394 397
@@ -400,9 +403,9 @@ u32 nvgpu_nvlink_enumerate(struct gk20a *g)
400 return err; 403 return err;
401} 404}
402 405
403u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) 406int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off)
404{ 407{
405 u32 err = -ENODEV; 408 int err = -ENODEV;
406#ifdef CONFIG_TEGRA_NVLINK 409#ifdef CONFIG_TEGRA_NVLINK
407 struct nvlink_device *ndev; 410 struct nvlink_device *ndev;
408 411
@@ -423,10 +426,10 @@ u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off)
423 return err; 426 return err;
424} 427}
425 428
426u32 nvgpu_nvlink_probe(struct gk20a *g) 429int nvgpu_nvlink_probe(struct gk20a *g)
427{ 430{
428#ifdef CONFIG_TEGRA_NVLINK 431#ifdef CONFIG_TEGRA_NVLINK
429 u32 err = 0; 432 int err = 0;
430 struct device_node *np = nvgpu_get_node(g); 433 struct device_node *np = nvgpu_get_node(g);
431 struct device_node *nvlink_np = NULL, *endp_np = NULL; 434 struct device_node *nvlink_np = NULL, *endp_np = NULL;
432 struct nvlink_device *ndev; 435 struct nvlink_device *ndev;
@@ -435,13 +438,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g)
435 /* Parse DT */ 438 /* Parse DT */
436 if (np) { 439 if (np) {
437 nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); 440 nvlink_np = of_get_child_by_name(np, "nvidia,nvlink");
438 if (nvlink_np) 441 if (nvlink_np) {
439 endp_np = of_get_child_by_name(np, "endpoint"); 442 endp_np = of_get_child_by_name(nvlink_np, "endpoint");
443 }
440 } 444 }
441 445
442 if (!endp_np) { 446 if (!endp_np) {
443 nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, 447 nvgpu_info(g, "no nvlink DT detected");
444 "No Nvlink DT detected");
445 return -ENODEV; 448 return -ENODEV;
446 } 449 }
447 450
@@ -466,13 +469,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g)
466 of_property_read_u32(endp_np, "physical_link", 469 of_property_read_u32(endp_np, "physical_link",
467 &phys_link_id); 470 &phys_link_id);
468 471
469 g->nvlink.topology_connected_links = BIT(phys_link_id); 472 g->nvlink.connected_links = BIT(phys_link_id);
470 473
471 /* Check that we are in dGPU mode */ 474 /* Check that we are in dGPU mode */
472 if (ndev->device_id != NVLINK_ENDPT_GV100) { 475 if (ndev->device_id != NVLINK_ENDPT_GV100) {
473 nvgpu_err(g, "Local nvlink device is not dGPU"); 476 nvgpu_err(g, "Local nvlink device is not dGPU");
474 err = -EINVAL; 477 err = -EINVAL;
475 goto free_nvlink; 478 goto free_ndev;
476 } 479 }
477 480
478 /* Fill in device struct */ 481 /* Fill in device struct */
@@ -500,22 +503,27 @@ u32 nvgpu_nvlink_probe(struct gk20a *g)
500 err = nvlink_register_device(ndev); 503 err = nvlink_register_device(ndev);
501 if (err) { 504 if (err) {
502 nvgpu_err(g, "failed on nvlink device registration"); 505 nvgpu_err(g, "failed on nvlink device registration");
503 goto free_nvlink; 506 goto free_ndev;
504 } 507 }
505 508
506 /* Register link with core driver */ 509 /* Register link with core driver */
507 err = nvlink_register_link(&ndev->link); 510 err = nvlink_register_link(&ndev->link);
508 if (err) { 511 if (err) {
509 nvgpu_err(g, "failed on nvlink link registration"); 512 nvgpu_err(g, "failed on nvlink link registration");
510 goto free_nvlink; 513 goto unregister_ndev;
511 } 514 }
512 515
513 /* Enable NVLINK support */ 516 /* Enable NVLINK support */
514 __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); 517 __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true);
515free_nvlink: 518 return 0;
519
520unregister_ndev:
521 nvlink_unregister_device(ndev);
522
523free_ndev:
516 nvgpu_kfree(g, ndev); 524 nvgpu_kfree(g, ndev);
525 g->nvlink.priv = NULL;
517 return err; 526 return err;
518
519#else 527#else
520 return -ENODEV; 528 return -ENODEV;
521#endif 529#endif
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h
index 02c7d0d9..40656edd 100644
--- a/drivers/gpu/nvgpu/gk20a/gk20a.h
+++ b/drivers/gpu/nvgpu/gk20a/gk20a.h
@@ -967,6 +967,8 @@ struct gpu_ops {
967 void (*isr_stall)(struct gk20a *g); 967 void (*isr_stall)(struct gk20a *g);
968 bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); 968 bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr);
969 bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); 969 bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr);
970 bool (*is_stall_and_eng_intr_pending)(struct gk20a *g,
971 u32 act_eng_id);
970 u32 (*intr_stall)(struct gk20a *g); 972 u32 (*intr_stall)(struct gk20a *g);
971 void (*intr_stall_pause)(struct gk20a *g); 973 void (*intr_stall_pause)(struct gk20a *g);
972 void (*intr_stall_resume)(struct gk20a *g); 974 void (*intr_stall_resume)(struct gk20a *g);
@@ -1064,10 +1066,10 @@ struct gpu_ops {
1064 int (*check_priv_security)(struct gk20a *g); 1066 int (*check_priv_security)(struct gk20a *g);
1065 } fuse; 1067 } fuse;
1066 struct { 1068 struct {
1067 u32 (*init)(struct gk20a *g); 1069 int (*init)(struct gk20a *g);
1068 u32 (*discover_ioctrl)(struct gk20a *g); 1070 int (*discover_ioctrl)(struct gk20a *g);
1069 u32 (*discover_link)(struct gk20a *g); 1071 int (*discover_link)(struct gk20a *g);
1070 u32 (*isr)(struct gk20a *g); 1072 int (*isr)(struct gk20a *g);
1071 /* API */ 1073 /* API */
1072 int (*link_early_init)(struct gk20a *g, unsigned long mask); 1074 int (*link_early_init)(struct gk20a *g, unsigned long mask);
1073 u32 (*link_get_mode)(struct gk20a *g, u32 link_id); 1075 u32 (*link_get_mode)(struct gk20a *g, u32 link_id);
diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.c b/drivers/gpu/nvgpu/gv100/flcn_gv100.c
new file mode 100644
index 00000000..5820d6c9
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.c
@@ -0,0 +1,55 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22#include <nvgpu/falcon.h>
23#include <nvgpu/pmu.h>
24
25#include "gk20a/gk20a.h"
26#include "gk20a/flcn_gk20a.h"
27#include "gp106/flcn_gp106.h"
28#include "gv100/flcn_gv100.h"
29
30#include <nvgpu/hw/gv100/hw_falcon_gv100.h>
31
32void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn)
33{
34 struct gk20a *g = flcn->g;
35
36 switch (flcn->flcn_id) {
37 case FALCON_ID_MINION:
38 flcn->flcn_base = g->nvlink.minion_base;
39 flcn->is_falcon_supported = true;
40 flcn->is_interrupt_enabled = true;
41 break;
42 default:
43 break;
44 }
45
46 if (flcn->is_falcon_supported) {
47 nvgpu_mutex_init(&flcn->copy_lock);
48 gk20a_falcon_ops(flcn);
49 } else {
50 /*
51 * Fall back
52 */
53 gp106_falcon_hal_sw_init(flcn);
54 }
55}
diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.h b/drivers/gpu/nvgpu/gv100/flcn_gv100.h
new file mode 100644
index 00000000..869e74c8
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.h
@@ -0,0 +1,27 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22#ifndef __FLCN_GV100_H__
23#define __FLCN_GV100_H__
24
25void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn);
26
27#endif /* __FLCN_GV100_H__ */
diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c
index 6f4ab875..53d61bfb 100644
--- a/drivers/gpu/nvgpu/gv100/hal_gv100.c
+++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c
@@ -51,20 +51,19 @@
51#include "gm20b/pmu_gm20b.h" 51#include "gm20b/pmu_gm20b.h"
52#include "gm20b/acr_gm20b.h" 52#include "gm20b/acr_gm20b.h"
53 53
54#include "gp10b/fb_gp10b.h"
55#include "gp10b/gr_gp10b.h"
56
57#include "gp106/clk_gp106.h" 54#include "gp106/clk_gp106.h"
58#include "gp106/clk_arb_gp106.h" 55#include "gp106/clk_arb_gp106.h"
59#include "gp106/pmu_gp106.h" 56#include "gp106/pmu_gp106.h"
60#include "gp106/acr_gp106.h" 57#include "gp106/acr_gp106.h"
61#include "gp106/sec2_gp106.h" 58#include "gp106/sec2_gp106.h"
62#include "gp106/bios_gp106.h" 59#include "gp106/bios_gp106.h"
63#include "gv100/bios_gv100.h"
64#include "gp106/therm_gp106.h" 60#include "gp106/therm_gp106.h"
65#include "gp106/xve_gp106.h" 61#include "gp106/xve_gp106.h"
66#include "gp106/clk_gp106.h" 62#include "gp106/clk_gp106.h"
67#include "gp106/flcn_gp106.h" 63#include "gp106/flcn_gp106.h"
64
65#include "gp10b/fb_gp10b.h"
66#include "gp10b/gr_gp10b.h"
68#include "gp10b/ltc_gp10b.h" 67#include "gp10b/ltc_gp10b.h"
69#include "gp10b/therm_gp10b.h" 68#include "gp10b/therm_gp10b.h"
70#include "gp10b/mc_gp10b.h" 69#include "gp10b/mc_gp10b.h"
@@ -78,32 +77,33 @@
78#include "gv11b/css_gr_gv11b.h" 77#include "gv11b/css_gr_gv11b.h"
79#include "gv11b/dbg_gpu_gv11b.h" 78#include "gv11b/dbg_gpu_gv11b.h"
80#include "gv11b/hal_gv11b.h" 79#include "gv11b/hal_gv11b.h"
81#include "gv100/gr_gv100.h"
82#include "gv11b/gr_gv11b.h" 80#include "gv11b/gr_gv11b.h"
83#include "gv11b/mc_gv11b.h" 81#include "gv11b/mc_gv11b.h"
84#include "gv11b/ltc_gv11b.h" 82#include "gv11b/ltc_gv11b.h"
85#include "gv11b/gv11b.h" 83#include "gv11b/gv11b.h"
86#include "gv11b/ce_gv11b.h" 84#include "gv11b/ce_gv11b.h"
87#include "gv100/gr_ctx_gv100.h"
88#include "gv11b/mm_gv11b.h" 85#include "gv11b/mm_gv11b.h"
89#include "gv11b/pmu_gv11b.h" 86#include "gv11b/pmu_gv11b.h"
90#include "gv11b/fb_gv11b.h" 87#include "gv11b/fb_gv11b.h"
91#include "gv100/mm_gv100.h"
92#include "gv11b/pmu_gv11b.h" 88#include "gv11b/pmu_gv11b.h"
93#include "gv100/fb_gv100.h"
94#include "gv100/fifo_gv100.h"
95#include "gv11b/fifo_gv11b.h" 89#include "gv11b/fifo_gv11b.h"
96#include "gv11b/regops_gv11b.h" 90#include "gv11b/regops_gv11b.h"
97
98#include "gv11b/gv11b_gating_reglist.h" 91#include "gv11b/gv11b_gating_reglist.h"
99#include "gv100/regops_gv100.h"
100#include "gv11b/subctx_gv11b.h" 92#include "gv11b/subctx_gv11b.h"
101 93
102#include "gv100.h" 94#include "gv100.h"
103#include "hal_gv100.h" 95#include "hal_gv100.h"
96#include "gv100/bios_gv100.h"
104#include "gv100/fb_gv100.h" 97#include "gv100/fb_gv100.h"
98#include "gv100/fifo_gv100.h"
99#include "gv100/flcn_gv100.h"
100#include "gv100/gr_ctx_gv100.h"
101#include "gv100/gr_gv100.h"
102#include "gv100/mc_gv100.h"
105#include "gv100/mm_gv100.h" 103#include "gv100/mm_gv100.h"
106#include "gv100/pmu_gv100.h" 104#include "gv100/pmu_gv100.h"
105#include "gv100/nvlink_gv100.h"
106#include "gv100/regops_gv100.h"
107 107
108#include <nvgpu/bus.h> 108#include <nvgpu/bus.h>
109#include <nvgpu/debug.h> 109#include <nvgpu/debug.h>
@@ -651,7 +651,7 @@ static const struct gpu_ops gv100_ops = {
651 .apply_smpc_war = gv100_apply_smpc_war, 651 .apply_smpc_war = gv100_apply_smpc_war,
652 }, 652 },
653 .mc = { 653 .mc = {
654 .intr_enable = mc_gv11b_intr_enable, 654 .intr_enable = mc_gv100_intr_enable,
655 .intr_unit_config = mc_gp10b_intr_unit_config, 655 .intr_unit_config = mc_gp10b_intr_unit_config,
656 .isr_stall = mc_gp10b_isr_stall, 656 .isr_stall = mc_gp10b_isr_stall,
657 .intr_stall = mc_gp10b_intr_stall, 657 .intr_stall = mc_gp10b_intr_stall,
@@ -666,6 +666,9 @@ static const struct gpu_ops gv100_ops = {
666 .boot_0 = gk20a_mc_boot_0, 666 .boot_0 = gk20a_mc_boot_0,
667 .is_intr1_pending = mc_gp10b_is_intr1_pending, 667 .is_intr1_pending = mc_gp10b_is_intr1_pending,
668 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, 668 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending,
669 .is_intr_nvlink_pending = gv100_mc_is_intr_nvlink_pending,
670 .is_stall_and_eng_intr_pending =
671 gv100_mc_is_stall_and_eng_intr_pending,
669 }, 672 },
670 .debug = { 673 .debug = {
671 .show_dump = gk20a_debug_show_dump, 674 .show_dump = gk20a_debug_show_dump,
@@ -712,11 +715,30 @@ static const struct gpu_ops gv100_ops = {
712 .disable_shadow_rom = xve_disable_shadow_rom_gp106, 715 .disable_shadow_rom = xve_disable_shadow_rom_gp106,
713 }, 716 },
714 .falcon = { 717 .falcon = {
715 .falcon_hal_sw_init = gp106_falcon_hal_sw_init, 718 .falcon_hal_sw_init = gv100_falcon_hal_sw_init,
716 }, 719 },
717 .priv_ring = { 720 .priv_ring = {
718 .isr = gp10b_priv_ring_isr, 721 .isr = gp10b_priv_ring_isr,
719 }, 722 },
723 .nvlink = {
724 .discover_ioctrl = gv100_nvlink_discover_ioctrl,
725 .discover_link = gv100_nvlink_discover_link,
726 .init = gv100_nvlink_init,
727 .isr = gv100_nvlink_isr,
728 /* API */
729 .link_early_init = gv100_nvlink_link_early_init,
730 .link_get_state = gv100_nvlink_link_get_state,
731 .link_set_mode = gv100_nvlink_link_set_mode,
732 .link_get_mode = gv100_nvlink_link_get_mode,
733 .get_sublink_mode = gv100_nvlink_link_get_sublink_mode,
734 .get_tx_sublink_state = gv100_nvlink_link_get_tx_sublink_state,
735 .get_rx_sublink_state = gv100_nvlink_link_get_rx_sublink_state,
736 .set_sublink_mode = gv100_nvlink_link_set_sublink_mode,
737 .interface_init = gv100_nvlink_interface_init,
738 .reg_init = gv100_nvlink_reg_init,
739 .shutdown = gv100_nvlink_shutdown,
740 .early_init = gv100_nvlink_early_init,
741 },
720 .chip_init_gpu_characteristics = gv100_init_gpu_characteristics, 742 .chip_init_gpu_characteristics = gv100_init_gpu_characteristics,
721 .get_litter_value = gv100_get_litter_value, 743 .get_litter_value = gv100_get_litter_value,
722}; 744};
@@ -751,6 +773,7 @@ int gv100_init_hal(struct gk20a *g)
751 gops->xve = gv100_ops.xve; 773 gops->xve = gv100_ops.xve;
752 gops->falcon = gv100_ops.falcon; 774 gops->falcon = gv100_ops.falcon;
753 gops->priv_ring = gv100_ops.priv_ring; 775 gops->priv_ring = gv100_ops.priv_ring;
776 gops->nvlink = gv100_ops.nvlink;
754 777
755 /* clocks */ 778 /* clocks */
756 gops->clk.init_clk_support = gv100_ops.clk.init_clk_support; 779 gops->clk.init_clk_support = gv100_ops.clk.init_clk_support;
diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.c b/drivers/gpu/nvgpu/gv100/mc_gv100.c
new file mode 100644
index 00000000..a3f8ab06
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/mc_gv100.c
@@ -0,0 +1,94 @@
1/*
2 * GV100 master
3 *
4 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25#include <linux/types.h>
26
27#include "gk20a/gk20a.h"
28
29#include "gp10b/mc_gp10b.h"
30
31#include "mc_gv100.h"
32#include "gv11b/fb_gv11b.h"
33
34#include <nvgpu/hw/gv100/hw_mc_gv100.h>
35
36void mc_gv100_intr_enable(struct gk20a *g)
37{
38 u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g);
39
40 gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING),
41 0xffffffff);
42 gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING),
43 0xffffffff);
44 gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_ALL);
45
46 g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING] =
47 mc_intr_pfifo_pending_f() |
48 mc_intr_hub_pending_f() |
49 mc_intr_priv_ring_pending_f() |
50 mc_intr_pbus_pending_f() |
51 mc_intr_ltc_pending_f() |
52 mc_intr_nvlink_pending_f() |
53 eng_intr_mask;
54
55 g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] =
56 mc_intr_pfifo_pending_f()
57 | eng_intr_mask;
58
59 /* TODO: Enable PRI faults for HUB ECC err intr */
60 gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, g->mm.hub_intr_types);
61
62 gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING),
63 g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING]);
64
65 gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING),
66 g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]);
67
68}
69
70bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0)
71{
72 return ((mc_intr_0 & mc_intr_nvlink_pending_f()) ? true : false);
73}
74
75bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id)
76{
77 u32 mc_intr_0 = gk20a_readl(g, mc_intr_r(0));
78 u32 stall_intr, eng_intr_mask;
79
80 eng_intr_mask = gk20a_fifo_act_eng_interrupt_mask(g, act_eng_id);
81 if (mc_intr_0 & eng_intr_mask)
82 return true;
83
84 stall_intr = mc_intr_pfifo_pending_f() |
85 mc_intr_hub_pending_f() |
86 mc_intr_priv_ring_pending_f() |
87 mc_intr_pbus_pending_f() |
88 mc_intr_ltc_pending_f() |
89 mc_intr_nvlink_pending_f();
90 if (mc_intr_0 & stall_intr)
91 return true;
92
93 return false;
94}
diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.h b/drivers/gpu/nvgpu/gv100/mc_gv100.h
new file mode 100644
index 00000000..4aff4a36
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/mc_gv100.h
@@ -0,0 +1,30 @@
1/*
2 * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef MC_GV100_H
24#define MC_GV100_H
25struct gk20a;
26
27void mc_gv100_intr_enable(struct gk20a *g);
28bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0);
29bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id);
30#endif
diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c
new file mode 100644
index 00000000..e5e65c51
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c
@@ -0,0 +1,2654 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <nvgpu/nvgpu_common.h>
24#include <nvgpu/bios.h>
25#include <nvgpu/firmware.h>
26#include <nvgpu/bitops.h>
27#include <nvgpu/nvlink.h>
28#include <nvgpu/enabled.h>
29
30#include "gk20a/gk20a.h"
31#include "nvlink_gv100.h"
32
33#include <nvgpu/hw/gv100/hw_falcon_gv100.h>
34#include <nvgpu/hw/gv100/hw_top_gv100.h>
35#include <nvgpu/hw/gv100/hw_nvlinkip_discovery_gv100.h>
36#include <nvgpu/hw/gv100/hw_nvlipt_gv100.h>
37#include <nvgpu/hw/gv100/hw_ioctrl_gv100.h>
38#include <nvgpu/hw/gv100/hw_minion_gv100.h>
39#include <nvgpu/hw/gv100/hw_nvl_gv100.h>
40#include <nvgpu/hw/gv100/hw_ioctrlmif_gv100.h>
41#include <nvgpu/hw/gv100/hw_trim_gv100.h>
42#include <nvgpu/hw/gv100/hw_nvtlc_gv100.h>
43
44#include <nvgpu/hw/gv100/hw_fb_gv100.h>
45#include <nvgpu/hw/gv100/hw_mc_gv100.h>
46
47#define NVLINK_PLL_ON_TIMEOUT_MS 30
48#define NVLINK_SUBLINK_TIMEOUT_MS 200
49/*
50 * The manuals are missing some useful defines
51 * we add them for now
52 */
53
54#define IPT_INTR_CONTROL_LINK(i) (nvlipt_intr_control_link0_r() + (i)*4)
55#define IPT_ERR_UC_STATUS_LINK(i) (nvlipt_err_uc_status_link0_r() + (i)*36)
56#define IPT_ERR_UC_MASK_LINK(i) (nvlipt_err_uc_mask_link0_r() + (i)*36)
57#define IPT_ERR_UC_SEVERITY_LINK(i) (nvlipt_err_uc_severity_link0_r() + (i)*36)
58#define IPT_ERR_UC_FIRST_LINK(i) (nvlipt_err_uc_first_link0_r() + (i)*36)
59#define IPT_ERR_UC_ADVISORY_LINK(i) (nvlipt_err_uc_advisory_link0_r() + (i)*36)
60#define IPT_ERR_C_STATUS_LINK(i) (nvlipt_err_c_status_link0_r() + (i)*36)
61#define IPT_ERR_C_MASK_LINK(i) (nvlipt_err_c_mask_link0_r() + (i)*36)
62#define IPT_ERR_C_FIRST_LINK(i) (nvlipt_err_c_first_link0_r() + (i)*36)
63#define IPT_ERR_CONTROL_LINK(i) (nvlipt_err_control_link0_r() + (i)*4)
64
65#define IPT_ERR_UC_ACTIVE_BITS (nvlipt_err_uc_status_link0_dlprotocol_f(1) | \
66 nvlipt_err_uc_status_link0_datapoisoned_f(1) | \
67 nvlipt_err_uc_status_link0_flowcontrol_f(1) | \
68 nvlipt_err_uc_status_link0_responsetimeout_f(1) | \
69 nvlipt_err_uc_status_link0_targeterror_f(1) | \
70 nvlipt_err_uc_status_link0_unexpectedresponse_f(1) | \
71 nvlipt_err_uc_status_link0_receiveroverflow_f(1) | \
72 nvlipt_err_uc_status_link0_malformedpacket_f(1) | \
73 nvlipt_err_uc_status_link0_stompedpacketreceived_f(1) | \
74 nvlipt_err_uc_status_link0_unsupportedrequest_f(1) | \
75 nvlipt_err_uc_status_link0_ucinternal_f(1))
76
77
78#define MINION_FALCON_INTR_MASK (minion_falcon_irqmset_wdtmr_set_f() | \
79 minion_falcon_irqmset_halt_set_f() | \
80 minion_falcon_irqmset_exterr_set_f()| \
81 minion_falcon_irqmset_swgen0_set_f()| \
82 minion_falcon_irqmset_swgen1_set_f())
83
84#define MINION_FALCON_INTR_DEST ( \
85 minion_falcon_irqdest_host_wdtmr_host_f() | \
86 minion_falcon_irqdest_host_halt_host_f() | \
87 minion_falcon_irqdest_host_exterr_host_f() | \
88 minion_falcon_irqdest_host_swgen0_host_f() | \
89 minion_falcon_irqdest_host_swgen1_host_f() | \
90 minion_falcon_irqdest_target_wdtmr_host_normal_f() | \
91 minion_falcon_irqdest_target_halt_host_normal_f() | \
92 minion_falcon_irqdest_target_exterr_host_normal_f() | \
93 minion_falcon_irqdest_target_swgen0_host_normal_f() | \
94 minion_falcon_irqdest_target_swgen1_host_normal_f())
95
96/* Minion FW header format */
97union gv100_minion_hdr {
98 struct {
99 u32 os_code_offset;
100 u32 os_code_size;
101 u32 os_data_offset;
102 u32 os_data_size;
103 u32 num_apps;
104 u32 app_0_code_start;
105 };
106 u8 raw_data[256];
107};
108
109struct __nvlink_reginit {
110 u32 addr;
111 u32 value;
112};
113
114static const struct __nvlink_reginit __nvlink_reginit_per_link_tegra[] = {
115 /* NVTLC when connected to Tegra */
116 { 0x300U, 0x00800040U },
117 { 0x304U, 0x00000020U },
118 { 0x308U, 0x00000020U },
119 { 0x30CU, 0x00000020U },
120 { 0x310U, 0x00000020U },
121 { 0x314U, 0x00800040U },
122 { 0x318U, 0x00000000U },
123 { 0x31CU, 0x00000000U },
124 { 0x200U, 0x007F003FU },
125 { 0x204U, 0x007F005FU },
126 { 0x208U, 0x007F007FU },
127 { 0x20CU, 0x007F009FU },
128 { 0x210U, 0x007F00BFU },
129 { 0x214U, 0x00FF003FU },
130 { 0x218U, 0x00FF003FU },
131 { 0x21CU, 0x00FF003FU },
132 { 0xB00U, 0x010000BEU },
133 { 0xB04U, 0x00000064U },
134 { 0xB08U, 0x00000000U },
135 { 0xB0CU, 0x00000020U },
136 { 0xB10U, 0x00000000U },
137 { 0xB14U, 0x010000BEU },
138 { 0xB18U, 0x00000000U },
139 { 0xB1CU, 0x00000000U },
140 { 0xA00U, 0x00FF00BDU },
141 { 0xA04U, 0x00FF0121U },
142 { 0xA08U, 0x00FF0121U },
143 { 0xA0CU, 0x00FF0141U },
144 { 0xA10U, 0x00FF0141U },
145 { 0xA14U, 0x01FF01FFU },
146 { 0xA18U, 0x01FF01FFU },
147 { 0xA1CU, 0x01FF01FFU },
148 { 0xF04U, 0x00FFFFFFU },
149 { 0xF0CU, 0x00FFFFFFU },
150 { 0xF1CU, 0x003FFFFFU },
151 { 0xF24U, 0x003FFFFFU },
152 { 0x704U, 0x003FFFFFU },
153 { 0x70CU, 0x003FFFFFU },
154 { 0x400U, 0x00000001U },
155 { 0xC00U, 0x00000001U },
156};
157
158static const struct __nvlink_reginit __nvlink_reginit_per_link_gpu[] = {
159 /* NVTLC for PEER GPU */
160 { 0x300U, 0x00800040U },
161 { 0x304U, 0x00000000U },
162 { 0x308U, 0x00000000U },
163 { 0x30CU, 0x00000000U },
164 { 0x310U, 0x00000000U },
165 { 0x314U, 0x00800040U },
166 { 0x318U, 0x00000000U },
167 { 0x31CU, 0x00000000U },
168 { 0x200U, 0x007F003FU },
169 { 0x204U, 0x007F003FU },
170 { 0x208U, 0x007F003FU },
171 { 0x20CU, 0x007F003FU },
172 { 0x210U, 0x007F003FU },
173 { 0x214U, 0x00FF007FU },
174 { 0x218U, 0x00FF007FU },
175 { 0x21CU, 0x00FF007FU },
176 { 0xB00U, 0x010000C0U },
177 { 0xB04U, 0x00000000U },
178 { 0xB08U, 0x00000000U },
179 { 0xB0CU, 0x00000000U },
180 { 0xB10U, 0x00000000U },
181 { 0xB14U, 0x010000C0U },
182 { 0xB18U, 0x00000000U },
183 { 0xB1CU, 0x00000000U },
184 { 0xA00U, 0x00FF00BFU },
185 { 0xA04U, 0x00FF00BFU },
186 { 0xA08U, 0x00FF00BFU },
187 { 0xA0CU, 0x00FF00BFU },
188 { 0xA10U, 0x00FF00BFU },
189 { 0xA14U, 0x01FF017FU },
190 { 0xA18U, 0x01FF017FU },
191 { 0xA1CU, 0x01FF017FU },
192 { 0xF04U, 0x00FFFFFFU },
193 { 0xF0CU, 0x00FFFFFFU },
194 { 0xF1CU, 0x003FFFFFU },
195 { 0xF24U, 0x003FFFFFU },
196 { 0x704U, 0x003FFFFFU },
197 { 0x70CU, 0x003FFFFFU },
198 { 0x400U, 0x00000001U },
199 { 0xC00U, 0x00000001U },
200};
201
202
203#define NVL_DEVICE(str) nvlinkip_discovery_common_device_##str##_v()
204
205const char * __gv100_device_type_to_str(u32 type)
206{
207 if (type == NVL_DEVICE(ioctrl))
208 return "IOCTRL";
209 if (type == NVL_DEVICE(dlpl))
210 return "DL/PL";
211 if (type == NVL_DEVICE(nvltlc))
212 return "NVLTLC";
213 if (type == NVL_DEVICE(ioctrlmif))
214 return "IOCTRLMIF";
215 if (type == NVL_DEVICE(nvlipt))
216 return "NVLIPT";
217 if (type == NVL_DEVICE(minion))
218 return "MINION";
219 if (type == NVL_DEVICE(dlpl_multicast))
220 return "DL/PL MULTICAST";
221 if (type == NVL_DEVICE(nvltlc_multicast))
222 return "NVLTLC MULTICAST";
223 if (type == NVL_DEVICE(ioctrlmif_multicast))
224 return "IOCTRLMIF MULTICAST";
225 if (type == NVL_DEVICE(nvltlc_multicast))
226 return "NVLTLC MULTICAST";
227 return "UNKNOWN";
228}
229
230/*
231 * Function prototypes
232 */
233static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask);
234static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g);
235static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask);
236static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g,
237 unsigned long mask, bool sync);
238
239
240/*
241 *******************************************************************************
242 * IP specific functions *
243 *******************************************************************************
244 */
245
246/*
247 *-----------------------------------------------------------------------------*
248 * MINION API
249 *-----------------------------------------------------------------------------*
250 */
251
252/*
253 * Initialization of link specific interrupts
254 */
255static void gv100_nvlink_minion_link_intr_enable(struct gk20a *g, u32 link_id,
256 bool enable)
257{
258 u32 intr, links;
259
260 /* Only stall interrupts for now */
261 intr = MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
262 links = minion_minion_intr_stall_en_link_v(intr);
263
264 if (enable)
265 links |= BIT(link_id);
266 else
267 links &= ~BIT(link_id);
268
269 intr = set_field(intr, minion_minion_intr_stall_en_link_m(),
270 minion_minion_intr_stall_en_link_f(links));
271 MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), intr);
272}
273
274/*
275 * Initialization of falcon interrupts
276 */
277static void gv100_nvlink_minion_falcon_intr_enable(struct gk20a *g, bool enable)
278{
279 u32 reg;
280
281 reg = MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
282 if (enable) {
283 reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(),
284 minion_minion_intr_stall_en_fatal_enable_f());
285 reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(),
286 minion_minion_intr_stall_en_nonfatal_enable_f());
287 reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(),
288 minion_minion_intr_stall_en_falcon_stall_enable_f());
289 reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(),
290 minion_minion_intr_stall_en_falcon_nostall_enable_f());
291 } else {
292 reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(),
293 minion_minion_intr_stall_en_fatal_disable_f());
294 reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(),
295 minion_minion_intr_stall_en_nonfatal_disable_f());
296 reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(),
297 minion_minion_intr_stall_en_falcon_stall_disable_f());
298 reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(),
299 minion_minion_intr_stall_en_falcon_nostall_disable_f());
300 }
301
302 MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), reg);
303}
304
305/*
306 * Initialize minion IP interrupts
307 */
308static void gv100_nvlink_initialize_minion(struct gk20a *g)
309{
310 /* Disable non-stall tree */
311 MINION_REG_WR32(g, minion_minion_intr_nonstall_en_r(), 0x0);
312
313 gv100_nvlink_minion_falcon_intr_enable(g, true);
314}
315
316/*
317 * Check if minion is up
318 */
319static bool __gv100_nvlink_minion_is_running(struct gk20a *g)
320{
321
322 /* if minion is booted and not halted, it is running */
323 if ((MINION_REG_RD32(g, minion_minion_status_r()) &
324 minion_minion_status_status_f(1)) &&
325 (!minion_falcon_irqstat_halt_v(
326 MINION_REG_RD32(g, minion_falcon_irqstat_r()))))
327 return true;
328
329 return false;
330}
331
332/*
333 * Falcon specific ISR handling
334 */
335static bool gv100_nvlink_minion_falcon_isr(struct gk20a *g)
336{
337 u32 intr;
338
339 intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) &
340 MINION_REG_RD32(g, minion_falcon_irqmask_r());
341
342 if (!intr)
343 return true;
344
345 if (intr & falcon_falcon_irqstat_exterr_true_f()) {
346 nvgpu_err(g, "FALCON EXT ADDR: 0x%x 0x%x 0x%x",
347 MINION_REG_RD32(g, 0x244),
348 MINION_REG_RD32(g, 0x248),
349 MINION_REG_RD32(g, 0x24c));
350 }
351
352 MINION_REG_WR32(g, minion_falcon_irqsclr_r(), intr);
353
354 nvgpu_err(g, "FATAL minion IRQ: 0x%08x", intr);
355
356
357 intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) &
358 MINION_REG_RD32(g, minion_falcon_irqmask_r());
359
360 return (intr == 0);
361}
362
363/*
364 * Link Specific ISR
365 */
366
367static bool gv100_nvlink_minion_link_isr(struct gk20a *g, u32 link_id)
368{
369 u32 intr, code;
370 bool fatal = false;
371
372 intr = MINION_REG_RD32(g, minion_nvlink_link_intr_r(link_id));
373 code = minion_nvlink_link_intr_code_v(intr);
374
375 if (code == minion_nvlink_link_intr_code_swreq_v()) {
376 nvgpu_err(g, " Intr SWREQ, link: %d subcode: %x",
377 link_id, minion_nvlink_link_intr_subcode_v(intr));
378 } else if (code == minion_nvlink_link_intr_code_pmdisabled_v()) {
379 nvgpu_err(g, " Fatal Intr PMDISABLED, link: %d subcode: %x",
380 link_id, minion_nvlink_link_intr_subcode_v(intr));
381 fatal = true;
382 } else if (code == minion_nvlink_link_intr_code_na_v()) {
383 nvgpu_err(g, " Fatal Intr NA, link: %d subcode: %x",
384 link_id, minion_nvlink_link_intr_subcode_v(intr));
385 fatal = true;
386 } else if (code == minion_nvlink_link_intr_code_dlreq_v()) {
387 nvgpu_err(g, " Fatal Intr DLREQ, link: %d subcode: %x",
388 link_id, minion_nvlink_link_intr_subcode_v(intr));
389 fatal = true;
390 } else {
391 nvgpu_err(g, " Fatal Intr UNKN:%x, link: %d subcode: %x", code,
392 link_id, minion_nvlink_link_intr_subcode_v(intr));
393 fatal = true;
394 }
395
396 if (fatal)
397 gv100_nvlink_minion_link_intr_enable(g, link_id, false);
398
399 intr = set_field(intr, minion_nvlink_link_intr_state_m(),
400 minion_nvlink_link_intr_state_f(1));
401 MINION_REG_WR32(g, minion_nvlink_link_intr_r(link_id), intr);
402
403 return true;
404}
405
406/*
407 * Global minion routine to service interrupts
408 */
409static bool gv100_nvlink_minion_isr(struct gk20a *g) {
410
411 u32 intr, i;
412 unsigned long links;
413
414 intr = MINION_REG_RD32(g, minion_minion_intr_r()) &
415 MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
416
417 if (minion_minion_intr_falcon_stall_v(intr) ||
418 minion_minion_intr_falcon_nostall_v(intr))
419 gv100_nvlink_minion_falcon_isr(g);
420
421 if (minion_minion_intr_fatal_v(intr)) {
422 gv100_nvlink_minion_falcon_intr_enable(g, false);
423 MINION_REG_WR32(g, minion_minion_intr_r(),
424 minion_minion_intr_fatal_f(1));
425 }
426
427 if (minion_minion_intr_nonfatal_v(intr))
428 MINION_REG_WR32(g, minion_minion_intr_r(),
429 minion_minion_intr_nonfatal_f(1));
430
431 links = minion_minion_intr_link_v(intr) & g->nvlink.enabled_links;
432
433 if (links)
434 for_each_set_bit(i, &links, 32)
435 gv100_nvlink_minion_link_isr(g, i);
436
437 /* Re-test interrupt status */
438 intr = MINION_REG_RD32(g, minion_minion_intr_r()) &
439 MINION_REG_RD32(g, minion_minion_intr_stall_en_r());
440
441 return (intr == 0);
442}
443
444/*
445 * Load minion FW and set up bootstrap
446 */
447static u32 gv100_nvlink_minion_load(struct gk20a *g)
448{
449 struct bin_hdr *hdr = NULL;
450 struct nvgpu_firmware *minion_fw = NULL;
451 union gv100_minion_hdr *minion_hdr;
452 u32 *minion_ucode = NULL;
453 u32 err = 0;
454 struct nvgpu_timeout timeout;
455 u32 delay = GR_IDLE_CHECK_DEFAULT;
456 u32 reg;
457 u32 app;
458
459 nvgpu_log_fn(g, " ");
460
461 if (__gv100_nvlink_minion_is_running(g))
462 return 0;
463
464 /* get mem unlock ucode binary */
465 minion_fw = nvgpu_request_firmware(g, "minion.bin", 0);
466 if (!minion_fw) {
467 nvgpu_err(g, "minion ucode get fail");
468 err = -ENOENT;
469 goto exit;
470 }
471
472 /* nvdec falcon reset */
473 nvgpu_flcn_reset(&g->minion_flcn);
474
475 hdr = (struct bin_hdr *) minion_fw->data;
476
477 minion_hdr = (union gv100_minion_hdr *) (minion_fw->data +
478 hdr->header_offset);
479 minion_ucode = (u32 *) (minion_fw->data + hdr->data_offset);
480
481 nvgpu_log(g, gpu_dbg_nvlink,
482 "os_code_offset: 0x%x", minion_hdr->os_code_offset);
483 nvgpu_log(g, gpu_dbg_nvlink,
484 "os_code_size: 0x%x", minion_hdr->os_code_size);
485 nvgpu_log(g, gpu_dbg_nvlink,
486 "os_data_offset: 0x%x", minion_hdr->os_data_offset);
487 nvgpu_log(g, gpu_dbg_nvlink,
488 "os_data_size: 0x%x", minion_hdr->os_data_size);
489 nvgpu_log(g, gpu_dbg_nvlink,
490 "num_apps: 0x%x", minion_hdr->num_apps);
491 nvgpu_log(g, gpu_dbg_nvlink,
492 "app_0_code_start: 0x%x", minion_hdr->app_0_code_start);
493
494 /* Clear interrupts */
495 nvgpu_flcn_set_irq(&g->minion_flcn, true, MINION_FALCON_INTR_MASK,
496 MINION_FALCON_INTR_DEST);
497
498 /* Copy Non Secure IMEM code */
499 nvgpu_flcn_copy_to_imem(&g->minion_flcn, 0,
500 (u8 *)&minion_ucode[minion_hdr->os_code_offset >> 2],
501 minion_hdr->os_code_size, 0, false,
502 GET_IMEM_TAG(minion_hdr->os_code_offset));
503
504 /* Copy Non Secure DMEM code */
505 nvgpu_flcn_copy_to_dmem(&g->minion_flcn, 0,
506 (u8 *)&minion_ucode[minion_hdr->os_data_offset >> 2],
507 minion_hdr->os_data_size, 0);
508
509 /* If apps are there load them securely */
510 if (minion_hdr->num_apps) {
511 for (app = 0; app < minion_hdr->num_apps; app++) {
512 u32 app_code_start, app_code_size;
513 u32 app_data_start, app_data_size;
514
515 app_code_start =
516 *((u32 *) &minion_hdr->app_0_code_start +
517 2*app);
518 app_code_size =
519 *((u32 *) &minion_hdr->app_0_code_start +
520 2*app + 1);
521 app_data_start =
522 *((u32 *) &minion_hdr->app_0_code_start +
523 2*minion_hdr->num_apps + 2*app);
524 app_data_size =
525 *((u32 *) &minion_hdr->app_0_code_start +
526 2*minion_hdr->num_apps + 2*app + 1);
527
528 nvgpu_log(g, gpu_dbg_nvlink,
529 " app %d code_offset 0x%x", app, app_code_start);
530 nvgpu_log(g, gpu_dbg_nvlink,
531 " app %d code_size 0x%x", app, app_code_size);
532 nvgpu_log(g, gpu_dbg_nvlink,
533 " app %d data_offset 0x%x", app, app_data_start);
534 nvgpu_log(g, gpu_dbg_nvlink,
535 " app %d data_size 0x%x", app, app_data_size);
536
537 if (app_code_size)
538 nvgpu_flcn_copy_to_imem(&g->minion_flcn,
539 app_code_start,
540 (u8 *)&minion_ucode[
541 app_code_start >> 2],
542 app_code_size, 0, true,
543 GET_IMEM_TAG(app_code_start));
544
545 if (app_data_size)
546 nvgpu_flcn_copy_to_dmem(&g->minion_flcn,
547 app_data_start,
548 (u8 *)&minion_ucode[
549 app_data_start >> 2],
550 app_data_size, 0);
551 }
552 }
553
554 /* set BOOTVEC to start of non-secure code */
555 nvgpu_flcn_bootstrap(&g->minion_flcn, 0x0);
556
557 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
558 NVGPU_TIMER_CPU_TIMER);
559
560 do {
561 reg = MINION_REG_RD32(g, minion_minion_status_r());
562
563 if (minion_minion_status_status_v(reg)) {
564 /* Minion sequence completed, check status */
565 if (minion_minion_status_status_v(reg) !=
566 minion_minion_status_status_boot_v()) {
567 nvgpu_err(g, "MINION init sequence failed: 0x%x",
568 minion_minion_status_status_v(reg));
569 err = -EINVAL;
570
571 goto exit;
572 }
573
574 nvgpu_log(g, gpu_dbg_nvlink,
575 "MINION boot successful: 0x%x", reg);
576 err = 0;
577 break;
578 }
579
580 nvgpu_usleep_range(delay, delay * 2);
581 delay = min_t(unsigned long,
582 delay << 1, GR_IDLE_CHECK_MAX);
583 } while (!nvgpu_timeout_expired_msg(&timeout, " minion boot timeout"));
584
585 /* Service interrupts */
586 gv100_nvlink_minion_falcon_isr(g);
587
588 if (nvgpu_timeout_peek_expired(&timeout)) {
589 err = -ETIMEDOUT;
590 goto exit;
591 }
592
593 gv100_nvlink_initialize_minion(g);
594
595exit:
596 if (minion_fw)
597 nvgpu_release_firmware(g, minion_fw);
598
599 return err;
600}
601
602/*
603 * Check if MINION command is complete
604 */
605static u32 gv100_nvlink_minion_command_complete(struct gk20a *g, u32 link_id)
606{
607 u32 reg;
608 struct nvgpu_timeout timeout;
609 u32 delay = GR_IDLE_CHECK_DEFAULT;
610
611
612 nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g),
613 NVGPU_TIMER_CPU_TIMER);
614
615 do {
616 reg = MINION_REG_RD32(g, minion_nvlink_dl_cmd_r(link_id));
617
618 if (minion_nvlink_dl_cmd_ready_v(reg) == 1) {
619 /* Command completed, check sucess */
620 if (minion_nvlink_dl_cmd_fault_v(reg) == 1) {
621 nvgpu_err(g, "minion cmd(%d) error: 0x%x",
622 link_id, reg);
623
624 reg = minion_nvlink_dl_cmd_fault_f(1);
625 MINION_REG_WR32(g,
626 minion_nvlink_dl_cmd_r(link_id), reg);
627
628 return -EINVAL;
629 }
630
631 /* Commnand success */
632 break;
633 }
634 nvgpu_usleep_range(delay, delay * 2);
635 delay = min_t(unsigned long,
636 delay << 1, GR_IDLE_CHECK_MAX);
637
638 } while (!nvgpu_timeout_expired_msg(&timeout, " minion cmd timeout"));
639
640 if (nvgpu_timeout_peek_expired(&timeout))
641 return -ETIMEDOUT;
642
643 nvgpu_log(g, gpu_dbg_nvlink, "minion cmd Complete");
644 return 0;
645}
646
647/*
648 * Send Minion command (can be async)
649 */
650static u32 gv100_nvlink_minion_send_command(struct gk20a *g,
651 u32 link_id, u32 command, u32 scratch_0, bool sync)
652{
653 u32 err = 0;
654
655 /* Check last command succeded */
656 err = gv100_nvlink_minion_command_complete(g, link_id);
657 if (err)
658 return -EINVAL;
659
660 nvgpu_log(g, gpu_dbg_nvlink,
661 "sending MINION command 0x%x to link %d", command, link_id);
662
663 if (command == minion_nvlink_dl_cmd_command_configeom_v())
664 MINION_REG_WR32(g, minion_misc_0_r(),
665 minion_misc_0_scratch_swrw_0_f(scratch_0));
666
667 MINION_REG_WR32(g, minion_nvlink_dl_cmd_r(link_id),
668 minion_nvlink_dl_cmd_command_f(command) |
669 minion_nvlink_dl_cmd_fault_f(1));
670
671 if (sync)
672 err = gv100_nvlink_minion_command_complete(g, link_id);
673
674 return err;
675}
676
677/* MINION API COMMANDS */
678
679/*
680 * Init UPHY
681 */
682static u32 gv100_nvlink_minion_init_uphy(struct gk20a *g, unsigned long mask,
683 bool sync)
684{
685 u32 err = 0;
686 u32 init_pll_cmd;
687 u32 link_id, master_pll, slave_pll;
688 u32 master_state, slave_state;
689
690 unsigned long link_enable;
691
692 switch(g->nvlink.speed) {
693 case nvgpu_nvlink_speed_default:
694 init_pll_cmd = minion_nvlink_dl_cmd_command_initpll_0_v();
695 break;
696 default:
697 nvgpu_err(g, "Unsupported UPHY speed");
698 return -EINVAL;
699 }
700
701 link_enable = __gv100_nvlink_get_link_reset_mask(g);
702
703 /* Cannot use links 0/1 without refclk buffer */
704 if (mask & (BIT(1)|BIT(0))) {
705 nvgpu_err(g, "links 0/1 not supported on GV100");
706 return -EINVAL;
707 }
708
709 for_each_set_bit(link_id, &mask, 32) {
710 master_pll = g->nvlink.links[link_id].pll_master_link_id;
711 slave_pll = g->nvlink.links[link_id].pll_slave_link_id;
712
713 master_state = nvl_link_state_state_init_v();
714 slave_state = nvl_link_state_state_init_v();
715
716 if (BIT(master_pll) & link_enable)
717 master_state = nvl_link_state_state_v(
718 g->ops.nvlink.link_get_state(g, master_pll));
719
720 if (BIT(slave_pll) & link_enable)
721 slave_state = nvl_link_state_state_v(
722 g->ops.nvlink.link_get_state(g, slave_pll));
723
724 if ((slave_state != nvl_link_state_state_init_v()) ||
725 (master_state != nvl_link_state_state_init_v())) {
726 nvgpu_err(g, "INIT PLL can only be executed when both "
727 "master and slave links are in init state");
728 return -EINVAL;
729 }
730
731 /* Check if INIT PLL is done on link */
732 if (!(BIT(master_pll) & g->nvlink.init_pll_done)) {
733 err = gv100_nvlink_minion_send_command(g, master_pll,
734 init_pll_cmd, 0, sync);
735 if (err) {
736 nvgpu_err(g, " Error sending INITPLL to minion");
737 return err;
738 }
739
740 g->nvlink.init_pll_done |= BIT(master_pll);
741 }
742 }
743
744 err = gv100_nvlink_setup_pll(g, mask);
745 if (err) {
746 nvgpu_err(g, "Error setting up PLL");
747 return err;
748 }
749
750 /* INITPHY commands */
751 for_each_set_bit(link_id, &mask, 32) {
752 err = gv100_nvlink_minion_send_command(g, link_id,
753 minion_nvlink_dl_cmd_command_initphy_v(), 0, sync);
754 if (err) {
755 nvgpu_err(g, "Error on INITPHY minion DL command %u",
756 link_id);
757 return err;
758 }
759 }
760
761 return 0;
762}
763
764/*
765 * Configure AC coupling
766 */
767static u32 gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g,
768 unsigned long mask, bool sync)
769{
770 u32 err = 0;
771 u32 i;
772 u32 temp;
773
774 for_each_set_bit(i, &mask, 32) {
775
776 temp = DLPL_REG_RD32(g, i, nvl_link_config_r());
777 temp &= ~nvl_link_config_ac_safe_en_m();
778 temp |= nvl_link_config_ac_safe_en_on_f();
779
780 DLPL_REG_WR32(g, i, nvl_link_config_r(), temp);
781
782 err = gv100_nvlink_minion_send_command(g, i,
783 minion_nvlink_dl_cmd_command_setacmode_v(), 0, sync);
784
785 if (err)
786 return err;
787 }
788
789 return err;
790}
791
792/*
793 * Set Data ready
794 */
795static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g,
796 unsigned long mask, bool sync)
797{
798 u32 err = 0;
799 u32 i;
800
801 for_each_set_bit(i, &mask, 32) {
802 err = gv100_nvlink_minion_send_command(g, i,
803 minion_nvlink_dl_cmd_command_initlaneenable_v(), 0,
804 sync);
805 if (err) {
806 nvgpu_err(g, "Failed init lane enable on minion");
807 return err;
808 }
809 }
810
811 for_each_set_bit(i, &mask, 32) {
812 err = gv100_nvlink_minion_send_command(g, i,
813 minion_nvlink_dl_cmd_command_initdlpl_v(), 0, sync);
814 if (err) {
815 nvgpu_err(g, "Failed init DLPL on minion");
816 return err;
817 }
818 }
819 return err;
820}
821
822/*
823 * Request that minion disable the lane
824 */
825static u32 gv100_nvlink_minion_lane_disable(struct gk20a *g, u32 link_id,
826 bool sync)
827{
828 u32 err = 0;
829
830 err = gv100_nvlink_minion_send_command(g, link_id,
831 minion_nvlink_dl_cmd_command_lanedisable_v(), 0, sync);
832
833 if (err)
834 nvgpu_err(g, " failed to disable lane on %d", link_id);
835
836 return err;
837}
838
839/*
840 * Request that minion shutdown the lane
841 */
842static u32 gv100_nvlink_minion_lane_shutdown(struct gk20a *g, u32 link_id,
843 bool sync)
844{
845 u32 err = 0;
846
847 err = gv100_nvlink_minion_send_command(g, link_id,
848 minion_nvlink_dl_cmd_command_laneshutdown_v(), 0, sync);
849
850 if (err)
851 nvgpu_err(g, " failed to shutdown lane on %d", link_id);
852
853 return err;
854}
855
856
857
858/*
859 *-----------------------------------------------------------------------------*
860 * TLC API
861 *-----------------------------------------------------------------------------*
862 */
863
864static int gv100_nvlink_get_tlc_reginit(enum nvgpu_nvlink_endp endp,
865 struct __nvlink_reginit **reg, u32 *count)
866{
867 switch(endp) {
868 case nvgpu_nvlink_endp_tegra:
869 *reg = (struct __nvlink_reginit *)
870 __nvlink_reginit_per_link_tegra;
871 *count = ARRAY_SIZE(__nvlink_reginit_per_link_tegra);
872 break;
873 case nvgpu_nvlink_endp_gpu:
874 *reg = (struct __nvlink_reginit *)
875 __nvlink_reginit_per_link_gpu;
876 *count = ARRAY_SIZE(__nvlink_reginit_per_link_gpu);
877 break;
878 default:
879 return -EINVAL;
880 }
881
882 return 0;
883}
884
885/*
886 * Init TLC IP and prod regs
887 */
888static void gv100_nvlink_initialize_tlc(struct gk20a *g, u32 link_id)
889{
890}
891
892/*
893 * Init TLC per link interrupts
894 */
895static void gv100_nvlink_tlc_intr_enable(struct gk20a *g, u32 link_id,
896 bool enable)
897{
898 u32 reg_rx0 = 0, reg_rx1 = 0, reg_tx = 0;
899
900 if (enable) {
901 /* Set PROD values */
902 reg_rx0 = 0x0FFFFFF;
903 reg_rx1 = 0x03FFFFF;
904 reg_tx = 0x1FFFFFF;
905 }
906
907 TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_0_r(), reg_rx0);
908 TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_1_r(), reg_rx1);
909 TLC_REG_WR32(g, link_id, nvtlc_tx_err_report_en_0_r(), reg_tx);
910}
911
912/*
913 * helper function to get TLC intr status in common structure
914 */
915static void gv100_nvlink_tlc_get_intr(struct gk20a *g, u32 link_id)
916{
917 g->nvlink.tlc_rx_err_status_0[link_id] =
918 TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_0_r());
919 g->nvlink.tlc_rx_err_status_1[link_id] =
920 TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_1_r());
921 g->nvlink.tlc_tx_err_status_0[link_id] =
922 TLC_REG_RD32(g, link_id, nvtlc_tx_err_status_0_r());
923}
924
925/*
926 * Interrupt routine handler for TLC
927 */
928static void gv100_nvlink_tlc_isr(struct gk20a *g, u32 link_id)
929{
930
931 if (g->nvlink.tlc_rx_err_status_0[link_id]) {
932 /* All TLC RX 0 errors are fatal. Notify and disable */
933 nvgpu_err(g, "Fatal TLC RX 0 interrupt on link %d mask: %x",
934 link_id, g->nvlink.tlc_rx_err_status_0[link_id]);
935 TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_0_r(),
936 g->nvlink.tlc_rx_err_status_0[link_id]);
937 TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_0_r(),
938 g->nvlink.tlc_rx_err_status_0[link_id]);
939 }
940 if (g->nvlink.tlc_rx_err_status_1[link_id]) {
941 /* All TLC RX 1 errors are fatal. Notify and disable */
942 nvgpu_err(g, "Fatal TLC RX 1 interrupt on link %d mask: %x",
943 link_id, g->nvlink.tlc_rx_err_status_1[link_id]);
944 TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_1_r(),
945 g->nvlink.tlc_rx_err_status_1[link_id]);
946 TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_1_r(),
947 g->nvlink.tlc_rx_err_status_1[link_id]);
948 }
949 if (g->nvlink.tlc_tx_err_status_0[link_id]) {
950 /* All TLC TX 0 errors are fatal. Notify and disable */
951 nvgpu_err(g, "Fatal TLC TX 0 interrupt on link %d mask: %x",
952 link_id, g->nvlink.tlc_tx_err_status_0[link_id]);
953 TLC_REG_WR32(g, link_id, nvtlc_tx_err_first_0_r(),
954 g->nvlink.tlc_tx_err_status_0[link_id]);
955 TLC_REG_WR32(g, link_id, nvtlc_tx_err_status_0_r(),
956 g->nvlink.tlc_tx_err_status_0[link_id]);
957 }
958}
959
960/*
961 *-----------------------------------------------------------------------------*
962 * DLPL API
963 *-----------------------------------------------------------------------------*
964 */
965
966/*
967 * DLPL interrupt enable helper
968 */
969static void gv100_nvlink_dlpl_intr_enable(struct gk20a *g, u32 link_id,
970 bool enable)
971{
972 u32 reg = 0;
973
974 /* Always disable nonstall tree */
975 DLPL_REG_WR32(g, link_id, nvl_intr_nonstall_en_r(), 0);
976
977 if (!enable)
978 {
979 DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), 0);
980 return;
981 }
982
983 /* Clear interrupt register to get rid of stale state (W1C) */
984 DLPL_REG_WR32(g, link_id, nvl_intr_r(), 0xffffffff);
985 DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff);
986
987 reg = nvl_intr_stall_en_tx_recovery_long_enable_f() |
988 nvl_intr_stall_en_tx_fault_ram_enable_f() |
989 nvl_intr_stall_en_tx_fault_interface_enable_f() |
990 nvl_intr_stall_en_tx_fault_sublink_change_enable_f() |
991 nvl_intr_stall_en_rx_fault_sublink_change_enable_f() |
992 nvl_intr_stall_en_rx_fault_dl_protocol_enable_f() |
993 nvl_intr_stall_en_ltssm_fault_enable_f();
994
995 DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), reg);
996
997 /* Configure error threshold */
998 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_error_rate_ctrl_r());
999 reg = set_field(reg, nvl_sl1_error_rate_ctrl_short_threshold_man_m(),
1000 nvl_sl1_error_rate_ctrl_short_threshold_man_f(0x2));
1001 reg = set_field(reg, nvl_sl1_error_rate_ctrl_long_threshold_man_m(),
1002 nvl_sl1_error_rate_ctrl_long_threshold_man_f(0x2));
1003 DLPL_REG_WR32(g, link_id, nvl_sl1_error_rate_ctrl_r(), reg);
1004}
1005
1006/*
1007 * DLPL per-link isr
1008 */
1009
1010#define DLPL_NON_FATAL_INTR_MASK (nvl_intr_tx_replay_f(1) | \
1011 nvl_intr_tx_recovery_short_f(1) | \
1012 nvl_intr_tx_recovery_long_f(1) | \
1013 nvl_intr_rx_short_error_rate_f(1) | \
1014 nvl_intr_rx_long_error_rate_f(1) | \
1015 nvl_intr_rx_ila_trigger_f(1) | \
1016 nvl_intr_ltssm_protocol_f(1))
1017
1018#define DLPL_FATAL_INTR_MASK ( nvl_intr_ltssm_fault_f(1) | \
1019 nvl_intr_rx_fault_dl_protocol_f(1) | \
1020 nvl_intr_rx_fault_sublink_change_f(1) | \
1021 nvl_intr_tx_fault_sublink_change_f(1) | \
1022 nvl_intr_tx_fault_interface_f(1) | \
1023 nvl_intr_tx_fault_ram_f(1))
1024
1025static void gv100_nvlink_dlpl_isr(struct gk20a *g, u32 link_id)
1026{
1027 u32 non_fatal_mask = 0;
1028 u32 fatal_mask = 0;
1029 u32 intr = 0;
1030 bool retrain = false;
1031 u32 err;
1032
1033 intr = DLPL_REG_RD32(g, link_id, nvl_intr_r()) &
1034 DLPL_REG_RD32(g, link_id, nvl_intr_stall_en_r());
1035
1036 if (!intr)
1037 return;
1038
1039 fatal_mask = intr & DLPL_FATAL_INTR_MASK;
1040 non_fatal_mask = intr & DLPL_NON_FATAL_INTR_MASK;
1041
1042 nvgpu_err(g, " handling DLPL %d isr. Fatal: %x non-Fatal: %x",
1043 link_id, fatal_mask, non_fatal_mask);
1044
1045 /* Check if we are not handling an interupt */
1046 if ((fatal_mask | non_fatal_mask) & ~intr)
1047 nvgpu_err(g, "Unable to service DLPL intr on link %d", link_id);
1048
1049 if (non_fatal_mask & nvl_intr_tx_recovery_long_f(1))
1050 retrain = true;
1051 if (fatal_mask)
1052 retrain = false;
1053
1054 if (retrain) {
1055 err = nvgpu_nvlink_train(g, link_id, false);
1056 if (err)
1057 nvgpu_err(g, "failed to retrain link %d", link_id);
1058 }
1059
1060 /* Clear interrupts */
1061 DLPL_REG_WR32(g, link_id, nvl_intr_r(), (non_fatal_mask | fatal_mask));
1062 DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff);
1063}
1064
1065/*
1066 *-----------------------------------------------------------------------------*
1067 * MIF API
1068 *-----------------------------------------------------------------------------*
1069 */
1070
1071/*
1072 * Initialize MIF API with PROD settings
1073 */
1074static void gv100_nvlink_initialize_mif(struct gk20a *g, u32 link_id)
1075{
1076 u32 tmp;
1077
1078 /* Enable MIF RX error */
1079
1080 /* Containment (make fatal) */
1081 tmp = 0;
1082 tmp = set_field(tmp,
1083 ioctrlmif_rx_err_contain_en_0_rxramdataparityerr_m(),
1084 ioctrlmif_rx_err_contain_en_0_rxramdataparityerr__prod_f());
1085 tmp = set_field(tmp,
1086 ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr_m(),
1087 ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr__prod_f());
1088 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_contain_en_0_r(), tmp);
1089
1090 /* Logging (do not ignore) */
1091 tmp = 0;
1092 tmp = set_field(tmp,
1093 ioctrlmif_rx_err_log_en_0_rxramdataparityerr_m(),
1094 ioctrlmif_rx_err_log_en_0_rxramdataparityerr_f(1));
1095 tmp = set_field(tmp,
1096 ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_m(),
1097 ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_f(1));
1098 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_log_en_0_r(), tmp);
1099
1100 /* Tx Error */
1101 /* Containment (make fatal) */
1102 tmp = 0;
1103 tmp = set_field(tmp,
1104 ioctrlmif_tx_err_contain_en_0_txramdataparityerr_m(),
1105 ioctrlmif_tx_err_contain_en_0_txramdataparityerr__prod_f());
1106 tmp = set_field(tmp,
1107 ioctrlmif_tx_err_contain_en_0_txramhdrparityerr_m(),
1108 ioctrlmif_tx_err_contain_en_0_txramhdrparityerr__prod_f());
1109 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_contain_en_0_r(), tmp);
1110
1111 /* Logging (do not ignore) */
1112 tmp = 0;
1113 tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramdataparityerr_m(),
1114 ioctrlmif_tx_err_log_en_0_txramdataparityerr_f(1));
1115 tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramhdrparityerr_m(),
1116 ioctrlmif_tx_err_log_en_0_txramhdrparityerr_f(1));
1117 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_log_en_0_r(), tmp);
1118
1119 /* Credit release */
1120 MIF_REG_WR32(g, link_id, ioctrlmif_rx_ctrl_buffer_ready_r(), 0x1);
1121 MIF_REG_WR32(g, link_id, ioctrlmif_tx_ctrl_buffer_ready_r(), 0x1);
1122}
1123
1124/*
1125 * Enable per-link MIF interrupts
1126 */
1127static void gv100_nvlink_mif_intr_enable(struct gk20a *g, u32 link_id,
1128 bool enable)
1129{
1130 u32 reg0 = 0, reg1 = 0;
1131
1132 if (enable) {
1133 reg0 = set_field(reg0,
1134 ioctrlmif_rx_err_report_en_0_rxramdataparityerr_m(),
1135 ioctrlmif_rx_err_report_en_0_rxramdataparityerr_f(1));
1136 reg0 = set_field(reg0,
1137 ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_m(),
1138 ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_f(1));
1139 reg1 = set_field(reg1,
1140 ioctrlmif_tx_err_report_en_0_txramdataparityerr_m(),
1141 ioctrlmif_tx_err_report_en_0_txramdataparityerr_f(1));
1142 reg1 = set_field(reg1,
1143 ioctrlmif_tx_err_report_en_0_txramhdrparityerr_m(),
1144 ioctrlmif_tx_err_report_en_0_txramhdrparityerr_f(1));
1145 }
1146
1147 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_report_en_0_r(), reg0);
1148 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_report_en_0_r(), reg1);
1149}
1150
1151/*
1152 * Handle per-link MIF interrupts
1153 */
1154static void gv100_nvlink_mif_isr(struct gk20a *g, u32 link_id)
1155{
1156 u32 intr, fatal_mask = 0;
1157
1158 /* RX Errors */
1159 intr = MIF_REG_RD32(g, link_id, ioctrlmif_rx_err_status_0_r());
1160 if (intr) {
1161 if (intr & ioctrlmif_rx_err_status_0_rxramdataparityerr_m()) {
1162 nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_DATA_PARITY",
1163 link_id);
1164 fatal_mask |= ioctrlmif_rx_err_status_0_rxramdataparityerr_f(1);
1165 }
1166 if (intr & ioctrlmif_rx_err_status_0_rxramhdrparityerr_m()) {
1167 nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_HDR_PARITY",
1168 link_id);
1169 fatal_mask |= ioctrlmif_rx_err_status_0_rxramhdrparityerr_f(1);
1170 }
1171
1172 if (fatal_mask) {
1173 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_first_0_r(),
1174 fatal_mask);
1175 MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_status_0_r(),
1176 fatal_mask);
1177 }
1178 }
1179
1180 /* TX Errors */
1181 fatal_mask = 0;
1182 intr = MIF_REG_RD32(g, link_id, ioctrlmif_tx_err_status_0_r());
1183 if (intr) {
1184 if (intr & ioctrlmif_tx_err_status_0_txramdataparityerr_m()) {
1185 nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_DATA_PARITY",
1186 link_id);
1187 fatal_mask |= ioctrlmif_tx_err_status_0_txramdataparityerr_f(1);
1188 }
1189 if (intr & ioctrlmif_tx_err_status_0_txramhdrparityerr_m()) {
1190 nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_HDR_PARITY",
1191 link_id);
1192 fatal_mask |= ioctrlmif_tx_err_status_0_txramhdrparityerr_f(1);
1193 }
1194
1195 if (fatal_mask) {
1196 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_first_0_r(),
1197 fatal_mask);
1198 MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_status_0_r(),
1199 fatal_mask);
1200 }
1201 }
1202}
1203
1204/*
1205 *-----------------------------------------------------------------------------*
1206 * NVLIPT API
1207 *-----------------------------------------------------------------------------*
1208 */
1209
1210/*
1211 * NVLIPT IP initialization (per-link)
1212 */
1213static void gv100_nvlink_initialize_nvlipt(struct gk20a *g, u32 link_id)
1214{
1215 /* init persistent scratch registers */
1216 IPT_REG_WR32(g, nvlipt_scratch_cold_r(),
1217 nvlipt_scratch_cold_data_init_v());
1218
1219 /*
1220 * AErr settings (top level)
1221 */
1222
1223 /* UC first and status reg (W1C) need to be cleared byt arch */
1224 IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1225 IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1226
1227 /* AErr Severity */
1228 IPT_REG_WR32(g, IPT_ERR_UC_SEVERITY_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1229
1230 /* AErr Control settings */
1231 IPT_REG_WR32(g, IPT_ERR_CONTROL_LINK(link_id),
1232 nvlipt_err_control_link0_fatalenable_f(1) |
1233 nvlipt_err_control_link0_nonfatalenable_f(1));
1234}
1235
1236/*
1237 * Enable NVLIPT interrupts
1238 */
1239static void gv100_nvlink_nvlipt_intr_enable(struct gk20a *g, u32 link_id,
1240 bool enable)
1241{
1242 u32 val = 0;
1243 u32 reg;
1244
1245 if (enable)
1246 val = 1;
1247
1248 reg = IPT_REG_RD32(g, IPT_INTR_CONTROL_LINK(link_id));
1249 reg = set_field(reg, nvlipt_intr_control_link0_stallenable_m(),
1250 nvlipt_intr_control_link0_stallenable_f(val));
1251 reg = set_field(reg, nvlipt_intr_control_link0_nostallenable_m(),
1252 nvlipt_intr_control_link0_nostallenable_f(val));
1253 IPT_REG_WR32(g, IPT_INTR_CONTROL_LINK(link_id), reg);
1254}
1255
1256/*
1257 * Per-link NVLIPT ISR handler
1258 */
1259bool gv100_nvlink_nvlipt_isr(struct gk20a *g, u32 link_id)
1260{
1261 /*
1262 * Interrupt handling happens in leaf handlers. Assume all interrupts
1263 * were handled and clear roll ups/
1264 */
1265 IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1266 IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS);
1267
1268 return true;
1269}
1270
1271/*
1272 *******************************************************************************
1273 * Interrupt handling functions *
1274 *******************************************************************************
1275 */
1276
1277/*
1278 * Enable common interrupts
1279 */
1280static void gv100_nvlink_common_intr_enable(struct gk20a *g,
1281 unsigned long mask)
1282{
1283 u32 reg, i;
1284
1285 /* Init HS HUB SW state */
1286 g->nvlink.hshub_config0 = gk20a_readl(g, fb_hshub_config0_r());
1287 g->nvlink.hshub_config1 = gk20a_readl(g, fb_hshub_config1_r());
1288 g->nvlink.hshub_config2 = gk20a_readl(g, fb_hshub_config2_r());
1289 g->nvlink.hshub_config6 = gk20a_readl(g, fb_hshub_config6_r());
1290
1291 /* Init IOCTRL */
1292 for_each_set_bit(i, &mask, 32) {
1293 reg = IOCTRL_REG_RD32(g, ioctrl_link_intr_0_mask_r(i));
1294 reg |= (ioctrl_link_intr_0_mask_fatal_f(1) |
1295 ioctrl_link_intr_0_mask_nonfatal_f(1) |
1296 ioctrl_link_intr_0_mask_correctable_f(1) |
1297 ioctrl_link_intr_0_mask_intra_f(1));
1298 IOCTRL_REG_WR32(g, ioctrl_link_intr_0_mask_r(i), reg);
1299 }
1300
1301 reg = IOCTRL_REG_RD32(g, ioctrl_common_intr_0_mask_r());
1302 reg |= (ioctrl_common_intr_0_mask_fatal_f(1) |
1303 ioctrl_common_intr_0_mask_nonfatal_f(1) |
1304 ioctrl_common_intr_0_mask_correctable_f(1) |
1305 ioctrl_common_intr_0_mask_intra_f(1));
1306 IOCTRL_REG_WR32(g, ioctrl_common_intr_0_mask_r(), reg);
1307
1308 /* Init NVLIPT */
1309 IPT_REG_WR32(g, nvlipt_intr_control_common_r(),
1310 nvlipt_intr_control_common_stallenable_f(1) |
1311 nvlipt_intr_control_common_nonstallenable_f(1));
1312}
1313
1314/*
1315 * Enable link specific interrupts (top-level)
1316 */
1317static void gv100_nvlink_enable_link_intr(struct gk20a *g, u32 link_id,
1318 bool enable)
1319{
1320 gv100_nvlink_minion_link_intr_enable(g, link_id, enable);
1321 gv100_nvlink_dlpl_intr_enable(g, link_id, enable);
1322 gv100_nvlink_tlc_intr_enable(g, link_id, enable);
1323 gv100_nvlink_mif_intr_enable(g, link_id, enable);
1324 gv100_nvlink_nvlipt_intr_enable(g, link_id, enable);
1325}
1326
1327/*
1328 * Top level interrupt handler
1329 */
1330int gv100_nvlink_isr(struct gk20a *g)
1331{
1332 unsigned long links;
1333 u32 link_id;
1334
1335 links = ioctrl_top_intr_0_status_link_v(
1336 IOCTRL_REG_RD32(g, ioctrl_top_intr_0_status_r()));
1337
1338 links &= g->nvlink.enabled_links;
1339 /* As per ARCH minion must be serviced first */
1340 gv100_nvlink_minion_isr(g);
1341
1342 for_each_set_bit(link_id, &links, 32) {
1343 /* Cache error logs from TLC, DL handler may clear them */
1344 gv100_nvlink_tlc_get_intr(g, link_id);
1345 gv100_nvlink_dlpl_isr(g, link_id);
1346 gv100_nvlink_tlc_isr(g, link_id);
1347 gv100_nvlink_mif_isr(g, link_id);
1348
1349 /* NVLIPT is top-level. Do it last */
1350 gv100_nvlink_nvlipt_isr(g, link_id);
1351 }
1352 return 0;
1353}
1354
1355/*******************************************************************************
1356 * Helper functions *
1357 *******************************************************************************
1358 */
1359
1360static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g)
1361{
1362 u32 reg_data;
1363
1364 reg_data = IOCTRL_REG_RD32(g, ioctrl_reset_r());
1365
1366 return ioctrl_reset_linkreset_v(reg_data);
1367}
1368
1369static u32 __gv100_nvlink_state_load_hal(struct gk20a *g)
1370{
1371 unsigned long discovered = g->nvlink.discovered_links;
1372
1373 gv100_nvlink_common_intr_enable(g, discovered);
1374 return gv100_nvlink_minion_load(g);
1375}
1376
1377#define TRIM_SYS_NVLINK_CTRL(i) (trim_sys_nvlink0_ctrl_r() + 16*i)
1378#define TRIM_SYS_NVLINK_STATUS(i) (trim_sys_nvlink0_status_r() + 16*i)
1379
1380static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask)
1381{
1382 u32 reg;
1383 u32 i;
1384 u32 links_off;
1385 struct nvgpu_timeout timeout;
1386 u32 pad_ctrl, swap_ctrl;
1387 u32 pll_id;
1388
1389 reg = gk20a_readl(g, trim_sys_nvlink_uphy_cfg_r());
1390 reg = set_field(reg, trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_m(),
1391 trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_f(1));
1392 gk20a_writel(g, trim_sys_nvlink_uphy_cfg_r(), reg);
1393
1394 reg = gk20a_readl(g, top_nvhsclk_ctrl_r());
1395
1396 pad_ctrl = top_nvhsclk_ctrl_e_clk_nvl_v(reg);
1397 swap_ctrl = top_nvhsclk_ctrl_swap_clk_nvl_v(reg);
1398
1399 for_each_set_bit(i, &mask, 32) {
1400 /* There are 3 PLLs for 6 links. We have 3 bits for each PLL.
1401 * The PLL bit corresponding to a link is /2 of its master link.
1402 */
1403 pll_id = g->nvlink.links[i].pll_master_link_id >> 1;
1404 pad_ctrl |= BIT(pll_id);
1405 swap_ctrl |= BIT(pll_id);
1406 }
1407
1408 reg = set_field(reg, top_nvhsclk_ctrl_e_clk_nvl_m(),
1409 top_nvhsclk_ctrl_e_clk_nvl_f(pad_ctrl));
1410 reg = set_field(reg, top_nvhsclk_ctrl_e_clk_core_m(),
1411 top_nvhsclk_ctrl_e_clk_core_f(0x1));
1412 reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_nvl_m(),
1413 top_nvhsclk_ctrl_swap_clk_nvl_f(swap_ctrl));
1414 reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_core_m(),
1415 top_nvhsclk_ctrl_swap_clk_core_f(0x0));
1416
1417 gk20a_writel(g, top_nvhsclk_ctrl_r(), reg);
1418
1419 for_each_set_bit(i, &mask, 32) {
1420 reg = gk20a_readl(g, TRIM_SYS_NVLINK_CTRL(i));
1421 reg = set_field(reg,
1422 trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_m(),
1423 trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_f(0));
1424 gk20a_writel(g, TRIM_SYS_NVLINK_CTRL(i), reg);
1425 }
1426
1427 /* Poll for links to go up */
1428 links_off = mask;
1429
1430 nvgpu_timeout_init(g, &timeout,
1431 NVLINK_PLL_ON_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
1432 do {
1433 for_each_set_bit(i, &mask, 32) {
1434 reg = gk20a_readl(g, TRIM_SYS_NVLINK_STATUS(i));
1435 if (trim_sys_nvlink0_status_pll_off_v(reg) == 0)
1436 links_off &= ~BIT(i);
1437 }
1438 nvgpu_udelay(5);
1439
1440 } while((!nvgpu_timeout_expired_msg(&timeout, "timeout on pll on")) &&
1441 links_off);
1442
1443 if (nvgpu_timeout_peek_expired(&timeout))
1444 return -ETIMEDOUT;
1445
1446 return 0;
1447}
1448
1449static void gv100_nvlink_prog_alt_clk(struct gk20a *g)
1450{
1451 u32 tmp;
1452
1453 /* RMW registers need to be separate */
1454 tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r());
1455 tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m();
1456 tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f();
1457 gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp);
1458
1459 tmp &= ~trim_sys_nvl_common_clk_alt_switch_finalsel_m();
1460 tmp |= trim_sys_nvl_common_clk_alt_switch_finalsel_slowclk_f();
1461}
1462
1463static int gv100_nvlink_enable_links_pre_top(struct gk20a *g, u32 links)
1464{
1465 u32 link_id;
1466 unsigned long enabled_links = links;
1467 u32 tmp;
1468 u32 reg;
1469 u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v();
1470 u32 err;
1471
1472 nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", enabled_links);
1473 /* Take links out of reset */
1474 for_each_set_bit(link_id, &enabled_links, 32) {
1475 reg = IOCTRL_REG_RD32(g, ioctrl_reset_r());
1476
1477 tmp = (BIT(link_id) |
1478 BIT(g->nvlink.links[link_id].pll_master_link_id));
1479
1480 reg = set_field(reg, ioctrl_reset_linkreset_m(),
1481 ioctrl_reset_linkreset_f( ioctrl_reset_linkreset_v(reg) |
1482 tmp));
1483
1484 IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg);
1485 nvgpu_udelay(delay);
1486
1487 reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r());
1488
1489 reg &= ~ioctrl_debug_reset_link_f(BIT(link_id));
1490 IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
1491 nvgpu_udelay(delay);
1492
1493 reg |= ioctrl_debug_reset_link_f(BIT(link_id));
1494 IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg);
1495 nvgpu_udelay(delay);
1496
1497 /* Enable Link DLPL for AN0 */
1498 reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r());
1499 reg = set_field(reg, nvl_link_config_link_en_m(),
1500 nvl_link_config_link_en_f(1));
1501 DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg);
1502
1503 /* This should be done by the NVLINK API */
1504 err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true);
1505 if (err) {
1506 nvgpu_err(g, "Failed to init phy of link: %u", link_id);
1507 return err;
1508 }
1509
1510 err = gv100_nvlink_rxcal_en(g, BIT(link_id));
1511 if (err) {
1512 nvgpu_err(g, "Failed to RXcal on link: %u", link_id);
1513 return err;
1514 }
1515
1516 err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true);
1517 if (err) {
1518 nvgpu_err(g, "Failed to set data ready link:%u",
1519 link_id);
1520 return err;
1521 }
1522
1523 g->nvlink.enabled_links |= BIT(link_id);
1524 }
1525
1526 nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x",
1527 g->nvlink.enabled_links);
1528
1529 if (g->nvlink.enabled_links)
1530 return 0;
1531
1532 nvgpu_err(g, " No links were enabled");
1533 return -EINVAL;
1534}
1535
1536static int gv100_nvlink_enable_links_post_top(struct gk20a *g, u32 links)
1537{
1538 u32 link_id;
1539 unsigned long enabled_links = (links & g->nvlink.enabled_links) &
1540 ~g->nvlink.initialized_links;
1541 u32 reg;
1542
1543 for_each_set_bit(link_id, &enabled_links, 32) {
1544
1545 /* WAR for HW bug 1888034 */
1546 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r());
1547 reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(),
1548 nvl_sl0_safe_ctrl2_tx_ctr_init_init_f());
1549 reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(),
1550 nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f());
1551 DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg);
1552
1553 gv100_nvlink_initialize_tlc(g, link_id);
1554 gv100_nvlink_initialize_nvlipt(g, link_id);
1555 gv100_nvlink_enable_link_intr(g, link_id, true);
1556
1557 g->nvlink.initialized_links |= BIT(link_id);
1558 };
1559
1560 return 0;
1561}
1562
1563static u32 gv100_nvlink_prbs_gen_en(struct gk20a *g, unsigned long mask)
1564{
1565 u32 reg;
1566 u32 link_id;
1567
1568 for_each_set_bit(link_id, &mask, 32) {
1569 /* Write is required as part of HW sequence */
1570 DLPL_REG_WR32(g, link_id, nvl_sl1_rxslsm_timeout_2_r(), 0);
1571
1572 reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r());
1573 reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(),
1574 nvl_txiobist_config_dpg_prbsseedld_f(0x1));
1575 DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg);
1576
1577 reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r());
1578 reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(),
1579 nvl_txiobist_config_dpg_prbsseedld_f(0x0));
1580 DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg);
1581 }
1582
1583 return 0;
1584}
1585
1586static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask)
1587{
1588 u32 link_id;
1589 struct nvgpu_timeout timeout;
1590 u32 reg;
1591
1592 for_each_set_bit(link_id, &mask, 32) {
1593 /* Timeout from HW specs */
1594 nvgpu_timeout_init(g, &timeout,
1595 8*NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
1596 reg = DLPL_REG_RD32(g, link_id, nvl_br0_cfg_cal_r());
1597 reg = set_field(reg, nvl_br0_cfg_cal_rxcal_m(),
1598 nvl_br0_cfg_cal_rxcal_on_f());
1599 DLPL_REG_WR32(g, link_id, nvl_br0_cfg_cal_r(), reg);
1600
1601 do {
1602 reg = DLPL_REG_RD32(g, link_id,
1603 nvl_br0_cfg_status_cal_r());
1604
1605 if (nvl_br0_cfg_status_cal_rxcal_done_v(reg) == 1)
1606 break;
1607 nvgpu_udelay(5);
1608 } while(!nvgpu_timeout_expired_msg(&timeout,
1609 "timeout on rxcal"));
1610
1611 if (nvgpu_timeout_peek_expired(&timeout))
1612 return -ETIMEDOUT;
1613 }
1614
1615 return 0;
1616}
1617
1618/*
1619 *******************************************************************************
1620 * Internal "ops" functions *
1621 *******************************************************************************
1622 */
1623
1624
1625/*
1626 * Main Nvlink init function. Calls into the Nvlink core API
1627 */
1628int gv100_nvlink_init(struct gk20a *g)
1629{
1630 int err = 0;
1631
1632 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK))
1633 return -ENODEV;
1634
1635 err = nvgpu_nvlink_enumerate(g);
1636 if (err)
1637 return err;
1638
1639 /* Set HSHUB and SG_PHY */
1640 __nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true);
1641 return err;
1642}
1643
1644/*
1645 * Query internal device topology and discover devices in nvlink local
1646 * infrastructure. Initialize register base and offsets
1647 */
1648int gv100_nvlink_discover_link(struct gk20a *g)
1649{
1650 u32 i;
1651 u32 ioctrl_entry_addr;
1652 u8 ioctrl_device_type;
1653 u32 table_entry;
1654 u32 ioctrl_info_entry_type;
1655 u8 ioctrl_discovery_size;
1656 bool is_chain = false;
1657 u8 nvlink_num_devices = 0;
1658 unsigned long available_links = 0;
1659 struct nvgpu_nvlink_device_list *device_table;
1660 u32 err = 0;
1661
1662 /*
1663 * Process Entry 0 & 1 of IOCTRL table to find table size
1664 */
1665 if (g->nvlink.ioctrl_table && g->nvlink.ioctrl_table[0].pri_base_addr) {
1666 ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr;
1667 table_entry = gk20a_readl(g, ioctrl_entry_addr);
1668 ioctrl_info_entry_type = nvlinkip_discovery_common_device_v(table_entry);
1669 } else {
1670 nvgpu_err(g, " Bad IOCTRL PRI Base addr");
1671 return -EINVAL;
1672 }
1673
1674 if (ioctrl_info_entry_type == NVL_DEVICE(ioctrl)) {
1675 ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr + 4;
1676 table_entry = gk20a_readl(g, ioctrl_entry_addr);
1677 ioctrl_discovery_size = nvlinkip_discovery_common_ioctrl_length_v(table_entry);
1678 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL size: %d", ioctrl_discovery_size);
1679 } else {
1680 nvgpu_err(g, " First entry of IOCTRL_DISCOVERY invalid");
1681 return -EINVAL;
1682 }
1683
1684 device_table = nvgpu_kzalloc(g, ioctrl_discovery_size *
1685 sizeof(struct nvgpu_nvlink_device_list));
1686 if (!device_table) {
1687 nvgpu_err(g, " Unable to allocate nvlink device table");
1688 return -ENOMEM;
1689 }
1690
1691 for (i = 0; i < ioctrl_discovery_size; i++) {
1692 ioctrl_entry_addr =
1693 g->nvlink.ioctrl_table[0].pri_base_addr + 4*i;
1694 table_entry = gk20a_readl(g, ioctrl_entry_addr);
1695
1696 nvgpu_log(g, gpu_dbg_nvlink, "parsing ioctrl %d: 0x%08x", i, table_entry);
1697
1698 ioctrl_info_entry_type = nvlinkip_discovery_common_entry_v(table_entry);
1699
1700 if (ioctrl_info_entry_type ==
1701 nvlinkip_discovery_common_entry_invalid_v())
1702 continue;
1703
1704 if (ioctrl_info_entry_type ==
1705 nvlinkip_discovery_common_entry_enum_v()) {
1706
1707 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is ENUM", i);
1708
1709 ioctrl_device_type =
1710 nvlinkip_discovery_common_device_v(table_entry);
1711
1712 if (nvlinkip_discovery_common_chain_v(table_entry) !=
1713 nvlinkip_discovery_common_chain_enable_v()) {
1714
1715 nvgpu_log(g, gpu_dbg_nvlink,
1716 "IOCTRL entry %d is ENUM but no chain",
1717 i);
1718 err = -EINVAL;
1719 break;
1720 }
1721
1722 is_chain = true;
1723 device_table[nvlink_num_devices].valid = true;
1724 device_table[nvlink_num_devices].device_type =
1725 ioctrl_device_type;
1726 device_table[nvlink_num_devices].device_id =
1727 nvlinkip_discovery_common_id_v(table_entry);
1728 device_table[nvlink_num_devices].device_version =
1729 nvlinkip_discovery_common_version_v(
1730 table_entry);
1731 continue;
1732 }
1733
1734 if (ioctrl_info_entry_type ==
1735 nvlinkip_discovery_common_entry_data1_v()) {
1736 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA1", i);
1737
1738 if (is_chain) {
1739 device_table[nvlink_num_devices].pri_base_addr =
1740 nvlinkip_discovery_common_pri_base_v(
1741 table_entry) << 12;
1742
1743 device_table[nvlink_num_devices].intr_enum =
1744 nvlinkip_discovery_common_intr_v(
1745 table_entry);
1746
1747 device_table[nvlink_num_devices].reset_enum =
1748 nvlinkip_discovery_common_reset_v(
1749 table_entry);
1750
1751 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d type = %d base: 0x%08x intr: %d reset: %d",
1752 i,
1753 device_table[nvlink_num_devices].device_type,
1754 device_table[nvlink_num_devices].pri_base_addr,
1755 device_table[nvlink_num_devices].intr_enum,
1756 device_table[nvlink_num_devices].reset_enum);
1757
1758 if (device_table[nvlink_num_devices].device_type ==
1759 NVL_DEVICE(dlpl)) {
1760 device_table[nvlink_num_devices].num_tx =
1761 nvlinkip_discovery_common_dlpl_num_tx_v(table_entry);
1762 device_table[nvlink_num_devices].num_rx =
1763 nvlinkip_discovery_common_dlpl_num_rx_v(table_entry);
1764
1765 nvgpu_log(g, gpu_dbg_nvlink, "DLPL tx: %d rx: %d",
1766 device_table[nvlink_num_devices].num_tx,
1767 device_table[nvlink_num_devices].num_rx);
1768 }
1769
1770 if (nvlinkip_discovery_common_chain_v(table_entry) !=
1771 nvlinkip_discovery_common_chain_enable_v()) {
1772
1773 is_chain = false;
1774 nvlink_num_devices++;
1775 }
1776 }
1777 continue;
1778 }
1779
1780 if (ioctrl_info_entry_type ==
1781 nvlinkip_discovery_common_entry_data2_v()) {
1782
1783 nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA2", i);
1784
1785 if (is_chain) {
1786 if (nvlinkip_discovery_common_dlpl_data2_type_v(table_entry)) {
1787 device_table[nvlink_num_devices].pll_master =
1788 nvlinkip_discovery_common_dlpl_data2_master_v(table_entry);
1789 device_table[nvlink_num_devices].pll_master_id =
1790 nvlinkip_discovery_common_dlpl_data2_masterid_v(table_entry);
1791 nvgpu_log(g, gpu_dbg_nvlink, "PLL info: Master: %d, Master ID: %d",
1792 device_table[nvlink_num_devices].pll_master,
1793 device_table[nvlink_num_devices].pll_master_id);
1794 }
1795
1796 if (nvlinkip_discovery_common_chain_v(table_entry) !=
1797 nvlinkip_discovery_common_chain_enable_v()) {
1798
1799 is_chain = false;
1800 nvlink_num_devices++;
1801 }
1802 }
1803 continue;
1804 }
1805 }
1806
1807 g->nvlink.device_table = device_table;
1808 g->nvlink.num_devices = nvlink_num_devices;
1809
1810 /*
1811 * Print table
1812 */
1813 for (i = 0; i < nvlink_num_devices; i++) {
1814 if (device_table[i].valid) {
1815 nvgpu_log(g, gpu_dbg_nvlink, "Device %d - %s", i,
1816 __gv100_device_type_to_str(
1817 device_table[i].device_type));
1818 nvgpu_log(g, gpu_dbg_nvlink, "+Link/Device Id: %d", device_table[i].device_id);
1819 nvgpu_log(g, gpu_dbg_nvlink, "+Version: %d", device_table[i].device_version);
1820 nvgpu_log(g, gpu_dbg_nvlink, "+Base Addr: 0x%08x", device_table[i].pri_base_addr);
1821 nvgpu_log(g, gpu_dbg_nvlink, "+Intr Enum: %d", device_table[i].intr_enum);
1822 nvgpu_log(g, gpu_dbg_nvlink, "+Reset Enum: %d", device_table[i].reset_enum);
1823 if ((device_table[i].device_type == NVL_DEVICE(dlpl)) ||
1824 (device_table[i].device_type == NVL_DEVICE(nvlink))) {
1825 nvgpu_log(g, gpu_dbg_nvlink, "+TX: %d", device_table[i].num_tx);
1826 nvgpu_log(g, gpu_dbg_nvlink, "+RX: %d", device_table[i].num_rx);
1827 nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master: %d", device_table[i].pll_master);
1828 nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master ID: %d", device_table[i].pll_master_id);
1829 }
1830 }
1831 }
1832
1833 for (i = 0; i < nvlink_num_devices; i++) {
1834 if (device_table[i].valid) {
1835
1836 if (device_table[i].device_type == NVL_DEVICE(ioctrl)) {
1837
1838 g->nvlink.ioctrl_type =
1839 device_table[i].device_type;
1840 g->nvlink.ioctrl_base =
1841 device_table[i].pri_base_addr;
1842 continue;
1843 }
1844
1845 if (device_table[i].device_type == NVL_DEVICE(dlpl)) {
1846
1847 g->nvlink.dlpl_type =
1848 device_table[i].device_type;
1849 g->nvlink.dlpl_base[device_table[i].device_id] =
1850 device_table[i].pri_base_addr;
1851 g->nvlink.links[device_table[i].device_id].valid = true;
1852 g->nvlink.links[device_table[i].device_id].g = g;
1853 g->nvlink.links[device_table[i].device_id].dlpl_version =
1854 device_table[i].device_version;
1855 g->nvlink.links[device_table[i].device_id].dlpl_base =
1856 device_table[i].pri_base_addr;
1857 g->nvlink.links[device_table[i].device_id].intr_enum =
1858 device_table[i].intr_enum;
1859 g->nvlink.links[device_table[i].device_id].reset_enum =
1860 device_table[i].reset_enum;
1861 g->nvlink.links[device_table[i].device_id].link_id =
1862 device_table[i].device_id;
1863
1864 /* initiate the PLL master and slave link id to max */
1865 g->nvlink.links[device_table[i].device_id].pll_master_link_id =
1866 NVLINK_MAX_LINKS_SW;
1867 g->nvlink.links[device_table[i].device_id].pll_slave_link_id =
1868 NVLINK_MAX_LINKS_SW;
1869
1870 /* Update Pll master */
1871 if (device_table[i].pll_master)
1872 g->nvlink.links[device_table[i].device_id].pll_master_link_id =
1873 g->nvlink.links[device_table[i].device_id].link_id;
1874 else {
1875 g->nvlink.links[device_table[i].device_id].pll_master_link_id =
1876 device_table[i].pll_master_id;
1877 g->nvlink.links[device_table[i].device_id].pll_slave_link_id =
1878 g->nvlink.links[device_table[i].device_id].link_id;
1879 g->nvlink.links[device_table[i].pll_master_id].pll_slave_link_id =
1880 g->nvlink.links[device_table[i].device_id].link_id;
1881 }
1882
1883 available_links |= BIT(device_table[i].device_id);
1884 continue;
1885 }
1886
1887 if (device_table[i].device_type == NVL_DEVICE(nvltlc)) {
1888
1889 g->nvlink.tl_type = device_table[i].device_type;
1890 g->nvlink.tl_base[device_table[i].device_id] =
1891 device_table[i].pri_base_addr;
1892 g->nvlink.links[device_table[i].device_id].tl_base =
1893 device_table[i].pri_base_addr;
1894 g->nvlink.links[device_table[i].device_id].tl_version =
1895 device_table[i].device_version;
1896 continue;
1897 }
1898
1899 if (device_table[i].device_type == NVL_DEVICE(nvltlc)) {
1900
1901 g->nvlink.tl_type = device_table[i].device_type;
1902 g->nvlink.tl_base[device_table[i].device_id] =
1903 device_table[i].pri_base_addr;
1904 g->nvlink.links[device_table[i].device_id].tl_base =
1905 device_table[i].pri_base_addr;
1906 g->nvlink.links[device_table[i].device_id].tl_version =
1907 device_table[i].device_version;
1908 continue;
1909 }
1910
1911 if (device_table[i].device_type == NVL_DEVICE(ioctrlmif)) {
1912
1913 g->nvlink.mif_type = device_table[i].device_type;
1914 g->nvlink.mif_base[device_table[i].device_id] =
1915 device_table[i].pri_base_addr;
1916 g->nvlink.links[device_table[i].device_id].mif_base =
1917 device_table[i].pri_base_addr;
1918 g->nvlink.links[device_table[i].device_id].mif_version =
1919 device_table[i].device_version;
1920 continue;
1921 }
1922
1923 if (device_table[i].device_type == NVL_DEVICE(nvlipt)) {
1924
1925 g->nvlink.ipt_type =
1926 device_table[i].device_type;
1927 g->nvlink.ipt_base =
1928 device_table[i].pri_base_addr;
1929 g->nvlink.ipt_version =
1930 device_table[i].device_version;
1931 continue;
1932 }
1933
1934 if (device_table[i].device_type == NVL_DEVICE(minion)) {
1935
1936 g->nvlink.minion_type =
1937 device_table[i].device_type;
1938 g->nvlink.minion_base =
1939 device_table[i].pri_base_addr;
1940 g->nvlink.minion_version =
1941 device_table[i].device_version;
1942 continue;
1943 }
1944
1945 if (device_table[i].device_type == NVL_DEVICE(dlpl_multicast)) {
1946
1947 g->nvlink.dlpl_multicast_type =
1948 device_table[i].device_type;
1949 g->nvlink.dlpl_multicast_base =
1950 device_table[i].pri_base_addr;
1951 g->nvlink.dlpl_multicast_version =
1952 device_table[i].device_version;
1953 continue;
1954 }
1955 if (device_table[i].device_type == NVL_DEVICE(nvltlc_multicast)) {
1956
1957 g->nvlink.tl_multicast_type =
1958 device_table[i].device_type;
1959 g->nvlink.tl_multicast_base =
1960 device_table[i].pri_base_addr;
1961 g->nvlink.tl_multicast_version =
1962 device_table[i].device_version;
1963 continue;
1964 }
1965
1966 if (device_table[i].device_type == NVL_DEVICE(ioctrlmif_multicast)) {
1967
1968 g->nvlink.mif_multicast_type =
1969 device_table[i].device_type;
1970 g->nvlink.mif_multicast_base =
1971 device_table[i].pri_base_addr;
1972 g->nvlink.mif_multicast_version =
1973 device_table[i].device_version;
1974 continue;
1975 }
1976
1977 }
1978 }
1979
1980 g->nvlink.discovered_links = (u32) available_links;
1981
1982 nvgpu_log(g, gpu_dbg_nvlink, "Nvlink Tree:");
1983 nvgpu_log(g, gpu_dbg_nvlink, "+ Available Links: 0x%08lx", available_links);
1984 nvgpu_log(g, gpu_dbg_nvlink, "+ Per-Link Devices:");
1985
1986 for_each_set_bit(i, &available_links, 32) {
1987 nvgpu_log(g, gpu_dbg_nvlink, "-- Link %d Dl/Pl Base: 0x%08x TLC Base: 0x%08x MIF Base: 0x%08x",
1988 i, g->nvlink.dlpl_base[i], g->nvlink.tl_base[i], g->nvlink.mif_base[i]);
1989 }
1990
1991 nvgpu_log(g, gpu_dbg_nvlink, "+ IOCTRL Base: 0x%08x", g->nvlink.ioctrl_base);
1992 nvgpu_log(g, gpu_dbg_nvlink, "+ NVLIPT Base: 0x%08x", g->nvlink.ipt_base);
1993 nvgpu_log(g, gpu_dbg_nvlink, "+ MINION Base: 0x%08x", g->nvlink.minion_base);
1994 nvgpu_log(g, gpu_dbg_nvlink, "+ DLPL MCAST Base: 0x%08x", g->nvlink.dlpl_multicast_base);
1995 nvgpu_log(g, gpu_dbg_nvlink, "+ TLC MCAST Base: 0x%08x", g->nvlink.tl_multicast_base);
1996 nvgpu_log(g, gpu_dbg_nvlink, "+ MIF MCAST Base: 0x%08x", g->nvlink.mif_multicast_base);
1997
1998 if (!g->nvlink.minion_version) {
1999 nvgpu_err(g, "Unsupported MINION version");
2000
2001 nvgpu_kfree(g, device_table);
2002 g->nvlink.device_table = NULL;
2003 g->nvlink.num_devices = 0;
2004 return -EINVAL;
2005 }
2006
2007 return err;
2008}
2009
2010/*
2011 * Query IOCTRL for device discovery
2012 */
2013int gv100_nvlink_discover_ioctrl(struct gk20a *g)
2014{
2015 u32 i;
2016 struct nvgpu_nvlink_ioctrl_list *ioctrl_table;
2017 u32 table_entry;
2018 u32 devinfo_type;
2019 bool is_ioctrl = false;
2020 bool is_chain = false;
2021 u32 io_num_entries = 0;
2022
2023 ioctrl_table = nvgpu_kzalloc(g, top_device_info__size_1_v() *
2024 sizeof(struct nvgpu_nvlink_ioctrl_list));
2025
2026 if (!ioctrl_table) {
2027 nvgpu_err(g, "failed to allocate memory for nvlink io table");
2028 return -ENOMEM;
2029 }
2030
2031 for (i = 0; i < top_device_info__size_1_v(); i++) {
2032 table_entry = gk20a_readl(g, top_device_info_r(i));
2033
2034 devinfo_type = top_device_info_entry_v(table_entry);
2035
2036 if (devinfo_type == top_device_info_entry_not_valid_v())
2037 continue;
2038
2039 if (devinfo_type == top_device_info_entry_engine_type_v()) {
2040 if (top_device_info_type_enum_v(table_entry) ==
2041 top_device_info_type_enum_ioctrl_v())
2042 is_ioctrl = true;
2043 else {
2044 is_ioctrl = false;
2045 continue;
2046 }
2047
2048 if (top_device_info_chain_v(table_entry) !=
2049 top_device_info_chain_enable_v())
2050 break;
2051
2052 is_chain = true;
2053 ioctrl_table[io_num_entries].valid = true;
2054 continue;
2055 }
2056
2057 if (devinfo_type == top_device_info_entry_data_v()) {
2058 if (is_ioctrl && is_chain) {
2059
2060 ioctrl_table[io_num_entries].pri_base_addr =
2061 top_device_info_data_pri_base_v(table_entry) <<
2062 top_device_info_data_pri_base_align_v();
2063
2064 if (top_device_info_chain_v(table_entry) !=
2065 top_device_info_chain_enable_v()) {
2066 is_chain = false;
2067 io_num_entries++;
2068 }
2069 }
2070 continue;
2071 }
2072
2073 if (devinfo_type == top_device_info_entry_enum_v()) {
2074 if (is_ioctrl && is_chain) {
2075
2076 ioctrl_table[io_num_entries].intr_enum =
2077 top_device_info_intr_v(table_entry);
2078 ioctrl_table[io_num_entries].reset_enum =
2079 top_device_info_reset_v(table_entry);
2080
2081 if (top_device_info_chain_v(table_entry) !=
2082 top_device_info_chain_enable_v()) {
2083 is_chain = false;
2084 io_num_entries++;
2085 }
2086 }
2087 continue;
2088 }
2089 }
2090
2091 if (io_num_entries == 0 || !ioctrl_table[0].pri_base_addr) {
2092 nvgpu_err(g, "No NVLINK io found");
2093 nvgpu_kfree(g, ioctrl_table);
2094 return -EINVAL;
2095 }
2096
2097 g->nvlink.ioctrl_table = ioctrl_table;
2098 g->nvlink.io_num_entries = io_num_entries;
2099
2100
2101 for (i =0; i < io_num_entries; i++)
2102 nvgpu_log(g, gpu_dbg_nvlink,
2103 "Device %d : Pri Base Addr = 0x%0x Intr = %d Reset = %d",
2104 i, ioctrl_table[i].pri_base_addr, ioctrl_table[i].intr_enum,
2105 ioctrl_table[i].reset_enum);
2106
2107 return 0;
2108}
2109
2110/*
2111 *******************************************************************************
2112 * NVLINK API FUNCTIONS *
2113 *******************************************************************************
2114 */
2115
2116/*
2117 * Performs link level initialization like phy inits, AN0 and interrupts
2118 */
2119
2120int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask)
2121{
2122 int err;
2123
2124 err = gv100_nvlink_enable_links_pre_top(g, mask);
2125 if (err) {
2126 nvgpu_err(g, "Pre topology failed for links %lx", mask);
2127 return err;
2128 }
2129
2130 nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx",
2131 mask & g->nvlink.enabled_links);
2132 err = gv100_nvlink_enable_links_post_top(g, mask);
2133
2134 return err;
2135}
2136
2137/*
2138 * Performs memory interface initialization
2139 */
2140
2141int gv100_nvlink_interface_init(struct gk20a *g)
2142{
2143 unsigned long mask = g->nvlink.enabled_links;
2144 u32 link_id;
2145
2146 for_each_set_bit(link_id, &mask, 32) {
2147 gv100_nvlink_initialize_mif(g, link_id);
2148 gv100_nvlink_mif_intr_enable(g, link_id, true);
2149 }
2150
2151 return 0;
2152}
2153
2154int gv100_nvlink_reg_init(struct gk20a *g)
2155{
2156 u32 i = 0;
2157 u32 count = 0;
2158 struct __nvlink_reginit *reg;
2159 enum nvgpu_nvlink_endp endp;
2160 int err;
2161 u32 link_id;
2162 unsigned long mask = g->nvlink.enabled_links;
2163 struct nvgpu_nvlink_link *link;
2164
2165 /* Apply automated reg init flow for PROD settings */
2166 for_each_set_bit(link_id, &mask, 32) {
2167
2168 link = &g->nvlink.links[link_id];
2169 if (!link->remote_info.is_connected)
2170 continue;
2171
2172 endp = link->remote_info.device_type;
2173 err = gv100_nvlink_get_tlc_reginit(endp, &reg, &count);
2174 if (err) {
2175 nvgpu_err(g, "no reginit for endp=%u", endp);
2176 continue;
2177 }
2178
2179 for (i = 0; i < count; i++) {
2180 TLC_REG_WR32(g, link_id, reg->addr, reg->value);
2181 reg++;
2182 }
2183 }
2184 return 0;
2185}
2186
2187/*
2188 * Shutdown device. This should tear down Nvlink connection.
2189 * For now return.
2190 */
2191int gv100_nvlink_shutdown(struct gk20a *g)
2192{
2193 return 0;
2194}
2195
2196/*
2197 * Get link state
2198 */
2199u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id)
2200{
2201 return DLPL_REG_RD32(g, link_id, nvl_link_state_r()) &
2202 nvl_link_state_state_m();
2203}
2204
2205/* Get link mode */
2206u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id)
2207{
2208 u32 state;
2209 if (!(BIT(link_id) & g->nvlink.discovered_links))
2210 return nvgpu_nvlink_link__last;
2211
2212 state = nvl_link_state_state_v(
2213 g->ops.nvlink.link_get_state(g, link_id));
2214
2215 if (state == nvl_link_state_state_init_v())
2216 return nvgpu_nvlink_link_off;
2217 if (state == nvl_link_state_state_hwcfg_v())
2218 return nvgpu_nvlink_link_detect;
2219 if (state == nvl_link_state_state_swcfg_v())
2220 return nvgpu_nvlink_link_safe;
2221 if (state == nvl_link_state_state_active_v())
2222 return nvgpu_nvlink_link_hs;
2223 if (state == nvl_link_state_state_fault_v())
2224 return nvgpu_nvlink_link_fault;
2225 if ((state == nvl_link_state_state_rcvy_ac_v()) ||
2226 (state == nvl_link_state_state_rcvy_sw_v()) ||
2227 (state == nvl_link_state_state_rcvy_rx_v()))
2228 return nvgpu_nvlink_link_recovery;
2229
2230 return nvgpu_nvlink_link_off;
2231}
2232
2233/* Set Link mode */
2234int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode)
2235{
2236 u32 state;
2237 u32 reg;
2238 u32 err = 0;
2239
2240 nvgpu_log(g, gpu_dbg_nvlink, "link :%d, mode:%u", link_id, mode);
2241
2242 if (!(BIT(link_id) & g->nvlink.enabled_links))
2243 return -EINVAL;
2244
2245 state = nvl_link_state_state_v(
2246 g->ops.nvlink.link_get_state(g, link_id));
2247
2248 switch (mode) {
2249 case nvgpu_nvlink_link_safe:
2250 if (state == nvl_link_state_state_swcfg_v()) {
2251 nvgpu_warn(g, "link is already in safe mode");
2252 break;
2253 }
2254 if (state == nvl_link_state_state_hwcfg_v()) {
2255 nvgpu_warn(g, "link is transitioning to safe mode");
2256 break;
2257 }
2258
2259 if (state == nvl_link_state_state_init_v()) {
2260 /* Off to Safe transition */
2261 reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
2262 reg = set_field(reg, nvl_link_change_newstate_m(),
2263 nvl_link_change_newstate_hwcfg_f());
2264 reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
2265 nvl_link_change_oldstate_mask_dontcare_f());
2266 reg = set_field(reg, nvl_link_change_action_m(),
2267 nvl_link_change_action_ltssm_change_f());
2268 DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
2269 } else if (state == nvl_link_state_state_active_v()) {
2270 /* TODO:
2271 * Disable PM first since we are moving out active
2272 * state
2273 */
2274 reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
2275 reg = set_field(reg, nvl_link_change_newstate_m(),
2276 nvl_link_change_newstate_swcfg_f());
2277 reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
2278 nvl_link_change_oldstate_mask_dontcare_f());
2279 reg = set_field(reg, nvl_link_change_action_m(),
2280 nvl_link_change_action_ltssm_change_f());
2281 DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
2282 }
2283 break;
2284
2285 case nvgpu_nvlink_link_hs:
2286 if (state == nvl_link_state_state_active_v()) {
2287 nvgpu_err(g, "link is already in active mode");
2288 break;
2289 }
2290 if (state == nvl_link_state_state_init_v()) {
2291 nvgpu_err(g, "link cannot be taken from init state");
2292 return -EPERM;
2293 }
2294
2295 reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r());
2296 reg = set_field(reg, nvl_link_change_newstate_m(),
2297 nvl_link_change_newstate_active_f());
2298 reg = set_field(reg, nvl_link_change_oldstate_mask_m(),
2299 nvl_link_change_oldstate_mask_dontcare_f());
2300 reg = set_field(reg, nvl_link_change_action_m(),
2301 nvl_link_change_action_ltssm_change_f());
2302 DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg);
2303 break;
2304
2305 case nvgpu_nvlink_link_off:
2306 if (state == nvl_link_state_state_active_v()) {
2307 nvgpu_err(g, "link cannot be taken from active to init");
2308 return -EPERM;
2309 }
2310 if (state == nvl_link_state_state_init_v()) {
2311 nvgpu_err(g, "link already in init state");
2312 }
2313
2314 /* GV100 UPHY is handled by MINION */
2315 break;
2316 /* 1/8 th mode not supported */
2317 case nvgpu_nvlink_link_enable_pm:
2318 case nvgpu_nvlink_link_disable_pm:
2319 return -EPERM;
2320 case nvgpu_nvlink_link_disable_err_detect:
2321 /* Disable Link interrupts */
2322 gv100_nvlink_dlpl_intr_enable(g, link_id, false);
2323 break;
2324 case nvgpu_nvlink_link_lane_disable:
2325 err = gv100_nvlink_minion_lane_disable(g, link_id, true);
2326 break;
2327 case nvgpu_nvlink_link_lane_shutdown:
2328 err = gv100_nvlink_minion_lane_shutdown(g, link_id, true);
2329 break;
2330 default:
2331 nvgpu_err(g, "Unhandled mode %x", mode);
2332 break;
2333 }
2334
2335 return err;
2336}
2337
2338static u32 gv100_nvlink_link_sublink_check_change(struct gk20a *g, u32 link_id)
2339{
2340 struct nvgpu_timeout timeout;
2341 u32 reg;
2342
2343 nvgpu_timeout_init(g, &timeout,
2344 NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER);
2345 /* Poll for sublink status */
2346 do {
2347 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2348
2349 if (nvl_sublink_change_status_v(reg) ==
2350 nvl_sublink_change_status_done_v())
2351 break;
2352 if (nvl_sublink_change_status_v(reg) ==
2353 nvl_sublink_change_status_fault_v()) {
2354 nvgpu_err(g, "Fault detected in sublink change");
2355 return -EFAULT;
2356 }
2357 nvgpu_udelay(5);
2358 } while(!nvgpu_timeout_expired_msg(&timeout, "timeout on sublink rdy"));
2359
2360 if (nvgpu_timeout_peek_expired(&timeout))
2361 return -ETIMEDOUT;
2362 return-0;
2363}
2364
2365int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id,
2366 bool is_rx_sublink, u32 mode)
2367{
2368 int err = 0;
2369 u32 rx_sublink_state, tx_sublink_state;
2370 u32 reg;
2371
2372 if (!(BIT(link_id) & g->nvlink.enabled_links))
2373 return -EINVAL;
2374
2375 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2376 if (err)
2377 return err;
2378
2379 rx_sublink_state = g->ops.nvlink.get_rx_sublink_state(g, link_id);
2380 tx_sublink_state = g->ops.nvlink.get_tx_sublink_state(g, link_id);
2381
2382 switch (mode) {
2383 case nvgpu_nvlink_sublink_tx_hs:
2384 if (tx_sublink_state ==
2385 nvl_sl0_slsm_status_tx_primary_state_hs_v()) {
2386 nvgpu_err(g, " TX already in HS");
2387 break;
2388 } else if (tx_sublink_state ==
2389 nvl_sl0_slsm_status_tx_primary_state_off_v()) {
2390 nvgpu_err(g, "TX cannot be do from OFF to HS");
2391 return -EPERM;
2392 }
2393
2394 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2395 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2396 nvl_sublink_change_newstate_hs_f());
2397 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2398 nvl_sublink_change_sublink_tx_f());
2399 reg = set_field(reg, nvl_sublink_change_action_m(),
2400 nvl_sublink_change_action_slsm_change_f());
2401 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2402
2403 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2404 if (err) {
2405 nvgpu_err(g, "Error in TX to HS");
2406 return err;
2407 }
2408 break;
2409 case nvgpu_nvlink_sublink_tx_common:
2410 err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true);
2411 break;
2412 case nvgpu_nvlink_sublink_tx_common_disable:
2413 /* NOP */
2414 break;
2415 case nvgpu_nvlink_sublink_tx_data_ready:
2416 err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true);
2417 break;
2418 case nvgpu_nvlink_sublink_tx_prbs_en:
2419 err = gv100_nvlink_prbs_gen_en(g, BIT(link_id));
2420 break;
2421 case nvgpu_nvlink_sublink_tx_safe:
2422 if (tx_sublink_state ==
2423 nvl_sl0_slsm_status_tx_primary_state_safe_v()) {
2424 nvgpu_err(g, "TX already SAFE: %d", link_id);
2425 break;
2426 }
2427
2428 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2429 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2430 nvl_sublink_change_newstate_safe_f());
2431 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2432 nvl_sublink_change_sublink_tx_f());
2433 reg = set_field(reg, nvl_sublink_change_action_m(),
2434 nvl_sublink_change_action_slsm_change_f());
2435 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2436
2437 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2438 if (err) {
2439 nvgpu_err(g, "Error in TX to SAFE");
2440 return err;
2441 }
2442 break;
2443 case nvgpu_nvlink_sublink_tx_off:
2444 if (tx_sublink_state ==
2445 nvl_sl0_slsm_status_tx_primary_state_off_v()) {
2446 nvgpu_err(g, "TX already OFF: %d", link_id);
2447 break;
2448 } else if (tx_sublink_state ==
2449 nvl_sl0_slsm_status_tx_primary_state_hs_v()) {
2450 nvgpu_err(g, " TX cannot go off from HS %d", link_id);
2451 return -EPERM;
2452 }
2453
2454 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2455 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2456 nvl_sublink_change_newstate_off_f());
2457 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2458 nvl_sublink_change_sublink_tx_f());
2459 reg = set_field(reg, nvl_sublink_change_action_m(),
2460 nvl_sublink_change_action_slsm_change_f());
2461 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2462
2463 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2464 if (err) {
2465 nvgpu_err(g, "Error in TX to OFF");
2466 return err;
2467 }
2468 break;
2469
2470 /* RX modes */
2471 case nvgpu_nvlink_sublink_rx_hs:
2472 case nvgpu_nvlink_sublink_rx_safe:
2473 break;
2474 case nvgpu_nvlink_sublink_rx_off:
2475 if (rx_sublink_state ==
2476 nvl_sl1_slsm_status_rx_primary_state_off_v()) {
2477 nvgpu_err(g, "RX already OFF: %d", link_id);
2478 break;
2479 } else if (rx_sublink_state ==
2480 nvl_sl1_slsm_status_rx_primary_state_hs_v()) {
2481 nvgpu_err(g, " RX cannot go off from HS %d", link_id);
2482 return -EPERM;
2483 }
2484
2485 reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r());
2486 reg = set_field(reg, nvl_sublink_change_newstate_m(),
2487 nvl_sublink_change_newstate_off_f());
2488 reg = set_field(reg, nvl_sublink_change_sublink_m(),
2489 nvl_sublink_change_sublink_rx_f());
2490 reg = set_field(reg, nvl_sublink_change_action_m(),
2491 nvl_sublink_change_action_slsm_change_f());
2492 DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg);
2493
2494 err = gv100_nvlink_link_sublink_check_change(g, link_id);
2495 if (err) {
2496 nvgpu_err(g, "Error in RX to OFF");
2497 return err;
2498 }
2499 break;
2500 case nvgpu_nvlink_sublink_rx_rxcal:
2501 err = gv100_nvlink_rxcal_en(g, BIT(link_id));
2502 break;
2503
2504 default:
2505 if ((is_rx_sublink) && ((mode < nvgpu_nvlink_sublink_rx_hs) ||
2506 (mode >= nvgpu_nvlink_sublink_rx__last))) {
2507 nvgpu_err(g, "Unsupported RX mode %u", mode);
2508 return -EINVAL;
2509 }
2510 if (mode >= nvgpu_nvlink_sublink_tx__last) {
2511 nvgpu_err(g, "Unsupported TX mode %u", mode);
2512 return -EINVAL;
2513 }
2514 nvgpu_err(g, "MODE %u", mode);
2515 }
2516
2517 if (err)
2518 nvgpu_err(g, " failed on set_sublink_mode");
2519 return err;
2520}
2521
2522u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id,
2523 bool is_rx_sublink)
2524{
2525 u32 state;
2526
2527 if (!(BIT(link_id) & g->nvlink.discovered_links)) {
2528 if (!is_rx_sublink)
2529 return nvgpu_nvlink_sublink_tx__last;
2530 return nvgpu_nvlink_sublink_rx__last;
2531 }
2532
2533 if (!is_rx_sublink) {
2534 state = g->ops.nvlink.get_tx_sublink_state(g, link_id);
2535 if (state == nvl_sl0_slsm_status_tx_primary_state_hs_v())
2536 return nvgpu_nvlink_sublink_tx_hs;
2537 if (state == nvl_sl0_slsm_status_tx_primary_state_eighth_v())
2538 return nvgpu_nvlink_sublink_tx_single_lane;
2539 if (state == nvl_sl0_slsm_status_tx_primary_state_safe_v())
2540 return nvgpu_nvlink_sublink_tx_safe;
2541 if (state == nvl_sl0_slsm_status_tx_primary_state_off_v())
2542 return nvgpu_nvlink_sublink_tx_off;
2543 return nvgpu_nvlink_sublink_tx_off;
2544 } else {
2545 state = g->ops.nvlink.get_rx_sublink_state(g, link_id);
2546 if (state == nvl_sl1_slsm_status_rx_primary_state_hs_v())
2547 return nvgpu_nvlink_sublink_rx_hs;
2548 if (state == nvl_sl1_slsm_status_rx_primary_state_eighth_v())
2549 return nvgpu_nvlink_sublink_rx_single_lane;
2550 if (state == nvl_sl1_slsm_status_rx_primary_state_safe_v())
2551 return nvgpu_nvlink_sublink_rx_safe;
2552 if (state == nvl_sl1_slsm_status_rx_primary_state_off_v())
2553 return nvgpu_nvlink_sublink_rx_off;
2554 return nvgpu_nvlink_sublink_rx_off;
2555 }
2556 return nvgpu_nvlink_sublink_tx__last;
2557}
2558
2559/*
2560 * Get TX sublink state
2561 */
2562u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id)
2563{
2564 u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_slsm_status_tx_r());
2565
2566 return nvl_sl0_slsm_status_tx_primary_state_v(reg);
2567}
2568
2569/*
2570 * Get RX sublink state
2571 */
2572u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id)
2573{
2574 u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_slsm_status_rx_r());
2575
2576 return nvl_sl1_slsm_status_rx_primary_state_v(reg);
2577}
2578
2579/*
2580 * Performs nvlink device level initialization by discovering the topology
2581 * taking device out of reset, boot minion, set clocks up and common interrupts
2582 */
2583int gv100_nvlink_early_init(struct gk20a *g)
2584{
2585 int err = 0;
2586
2587 if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK))
2588 return -EINVAL;
2589
2590 err = nvgpu_bios_get_nvlink_config_data(g);
2591 if (err) {
2592 nvgpu_err(g, "failed to read nvlink vbios data");
2593 goto nvlink_init_exit;
2594 }
2595
2596 err = g->ops.nvlink.discover_ioctrl(g);
2597 if (err)
2598 goto nvlink_init_exit;
2599
2600 /* Enable NVLINK in MC */
2601 g->ops.mc.reset(g, mc_enable_nvlink_enabled_f());
2602
2603 err = g->ops.nvlink.discover_link(g);
2604 if (err || g->nvlink.discovered_links == 0) {
2605 nvgpu_err(g, "No links available");
2606 goto nvlink_init_exit;
2607 }
2608
2609 nvgpu_flcn_sw_init(g, FALCON_ID_MINION);
2610
2611 g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask;
2612 nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)",
2613 g->nvlink.link_disable_mask);
2614
2615 /* Links in reset should be removed from initialized link sw state */
2616 g->nvlink.initialized_links &= __gv100_nvlink_get_link_reset_mask(g);
2617
2618 nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x (from DT)",
2619 g->nvlink.connected_links);
2620
2621 /* Track unconnected links */
2622 g->nvlink.discovered_links &= g->nvlink.connected_links;
2623
2624 nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)",
2625 g->nvlink.discovered_links);
2626
2627 if (hweight32(g->nvlink.discovered_links) > 1) {
2628 nvgpu_err(g, "more than one link enabled");
2629 err = -EINVAL;
2630 goto nvlink_init_exit;
2631 }
2632
2633 /* For now set default speed */
2634 g->nvlink.speed = nvgpu_nvlink_speed_default;
2635
2636 err = __gv100_nvlink_state_load_hal(g);
2637 if (err) {
2638 nvgpu_err(g, " failed Nvlink state load");
2639 goto nvlink_init_exit;
2640 }
2641 err = gv100_nvlink_minion_configure_ac_coupling(g,
2642 g->nvlink.ac_coupling_mask, true);
2643 if (err) {
2644 nvgpu_err(g, " failed Nvlink state load");
2645 goto nvlink_init_exit;
2646 }
2647
2648 /* Program clocks */
2649 gv100_nvlink_prog_alt_clk(g);
2650
2651nvlink_init_exit:
2652 return err;
2653}
2654
diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.h b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h
new file mode 100644
index 00000000..4ac8b907
--- /dev/null
+++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h
@@ -0,0 +1,60 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#ifndef NVGPU_NVLINK_GV100_H
24#define NVGPU_NVLINK_GV100_H
25
26struct gk20a;
27
28#define MINION_REG_RD32(g, off) gk20a_readl(g, g->nvlink.minion_base + (off))
29#define MINION_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.minion_base + (off), (v))
30#define IOCTRL_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ioctrl_base + (off))
31#define IOCTRL_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ioctrl_base + (off), (v));
32#define MIF_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].mif_base + (off))
33#define MIF_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].mif_base + (off), (v))
34#define IPT_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ipt_base + (off))
35#define IPT_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ipt_base + (off), (v))
36#define TLC_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].tl_base + (off))
37#define TLC_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].tl_base + (off), (v))
38#define DLPL_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].dlpl_base + (off))
39#define DLPL_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].dlpl_base + (off), (v))
40
41int gv100_nvlink_discover_ioctrl(struct gk20a *g);
42int gv100_nvlink_discover_link(struct gk20a *g);
43int gv100_nvlink_init(struct gk20a *g);
44int gv100_nvlink_isr(struct gk20a *g);
45/* API */
46int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask);
47u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id);
48u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id);
49int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode);
50u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id,
51 bool is_rx_sublink);
52u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id);
53u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id);
54int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id,
55 bool is_rx_sublink, u32 mode);
56int gv100_nvlink_interface_init(struct gk20a *g);
57int gv100_nvlink_reg_init(struct gk20a *g);
58int gv100_nvlink_shutdown(struct gk20a *g);
59int gv100_nvlink_early_init(struct gk20a *g);
60#endif
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
index 89f3f0b9..97ab7aab 100644
--- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c
@@ -482,7 +482,7 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id,
482 eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); 482 eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id));
483 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); 483 ctx_stat = fifo_engine_status_ctx_status_v(eng_stat);
484 484
485 if (gv11b_mc_is_stall_and_eng_intr_pending(g, act_eng_id)) { 485 if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id)) {
486 stall_intr = true; 486 stall_intr = true;
487 nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, 487 nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr,
488 "stall intr set, " 488 "stall intr set, "
diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
index 6fdac85a..5282af05 100644
--- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
+++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c
@@ -658,6 +658,8 @@ static const struct gpu_ops gv11b_ops = {
658 .boot_0 = gk20a_mc_boot_0, 658 .boot_0 = gk20a_mc_boot_0,
659 .is_intr1_pending = mc_gp10b_is_intr1_pending, 659 .is_intr1_pending = mc_gp10b_is_intr1_pending,
660 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, 660 .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending,
661 .is_stall_and_eng_intr_pending =
662 gv11b_mc_is_stall_and_eng_intr_pending,
661 }, 663 },
662 .debug = { 664 .debug = {
663 .show_dump = gk20a_debug_show_dump, 665 .show_dump = gk20a_debug_show_dump,
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h
index c59a6e6b..a57ea97a 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h
@@ -892,6 +892,14 @@ static inline u32 minion_nvlink_link_intr_code_dlreq_f(void)
892{ 892{
893 return 0x2U; 893 return 0x2U;
894} 894}
895static inline u32 minion_nvlink_link_intr_code_pmdisabled_v(void)
896{
897 return 0x00000003U;
898}
899static inline u32 minion_nvlink_link_intr_code_pmdisabled_f(void)
900{
901 return 0x3U;
902}
895static inline u32 minion_nvlink_link_intr_subcode_f(u32 v) 903static inline u32 minion_nvlink_link_intr_subcode_f(u32 v)
896{ 904{
897 return (v & 0xffU) << 8U; 905 return (v & 0xffU) << 8U;
diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h
index 48851ff1..18e0aab6 100644
--- a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h
+++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h
@@ -182,11 +182,10 @@ struct nvgpu_nvlink_dev {
182 u8 train_at_boot; 182 u8 train_at_boot;
183 u32 ac_coupling_mask; 183 u32 ac_coupling_mask;
184 184
185 u32 init_disabled_links;
186 u32 connected_links; 185 u32 connected_links;
187 u32 initialized_links; 186 u32 initialized_links;
188 u32 enabled_links; 187 u32 enabled_links;
189 u32 topology_connected_links; 188 u32 init_pll_done;
190 189
191 enum nvgpu_nvlink_speed speed; 190 enum nvgpu_nvlink_speed speed;
192 191
@@ -195,12 +194,18 @@ struct nvgpu_nvlink_dev {
195 u32 hshub_config1; 194 u32 hshub_config1;
196 u32 hshub_config2; 195 u32 hshub_config2;
197 u32 hshub_config6; 196 u32 hshub_config6;
197
198 /* tlc cached errors */
199 u32 tlc_rx_err_status_0[NVLINK_MAX_LINKS_SW];
200 u32 tlc_rx_err_status_1[NVLINK_MAX_LINKS_SW];
201 u32 tlc_tx_err_status_0[NVLINK_MAX_LINKS_SW];
202
198 /* priv struct */ 203 /* priv struct */
199 void *priv; 204 void *priv;
200}; 205};
201 206
202 207
203u32 nvgpu_nvlink_enumerate(struct gk20a *g); 208int nvgpu_nvlink_enumerate(struct gk20a *g);
204u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); 209int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off);
205u32 nvgpu_nvlink_probe(struct gk20a *g); 210int nvgpu_nvlink_probe(struct gk20a *g);
206#endif 211#endif
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-15 22:49:21 -0400