diff options
Diffstat (limited to 'drivers/gpu')
-rw-r--r-- | drivers/gpu/nvgpu/Makefile | 3 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/nvgpu_mem.c | 4 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/common/linux/nvlink.c | 46 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gk20a/gk20a.h | 10 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/flcn_gv100.c | 55 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/flcn_gv100.h | 27 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/hal_gv100.c | 49 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/mc_gv100.c | 94 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/mc_gv100.h | 30 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/nvlink_gv100.c | 2654 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv100/nvlink_gv100.h | 60 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv11b/fifo_gv11b.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/gv11b/hal_gv11b.c | 2 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h | 8 | ||||
-rw-r--r-- | drivers/gpu/nvgpu/include/nvgpu/nvlink.h | 15 |
15 files changed, 3016 insertions, 43 deletions
diff --git a/drivers/gpu/nvgpu/Makefile b/drivers/gpu/nvgpu/Makefile index 4169bc9b..d9d1da95 100644 --- a/drivers/gpu/nvgpu/Makefile +++ b/drivers/gpu/nvgpu/Makefile | |||
@@ -246,6 +246,9 @@ nvgpu-y += \ | |||
246 | gv100/fifo_gv100.o \ | 246 | gv100/fifo_gv100.o \ |
247 | gv100/gr_gv100.o \ | 247 | gv100/gr_gv100.o \ |
248 | gv100/regops_gv100.o \ | 248 | gv100/regops_gv100.o \ |
249 | gv100/flcn_gv100.o \ | ||
250 | gv100/mc_gv100.o \ | ||
251 | gv100/nvlink_gv100.o \ | ||
249 | gv100/hal_gv100.o \ | 252 | gv100/hal_gv100.o \ |
250 | gv100/pmu_gv100.o \ | 253 | gv100/pmu_gv100.o \ |
251 | pstate/pstate.o \ | 254 | pstate/pstate.o \ |
diff --git a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c index e4b4be06..206b83e1 100644 --- a/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c +++ b/drivers/gpu/nvgpu/common/linux/nvgpu_mem.c | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. | 2 | * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify it | 4 | * This program is free software; you can redistribute it and/or modify it |
5 | * under the terms and conditions of the GNU General Public License, | 5 | * under the terms and conditions of the GNU General Public License, |
@@ -536,6 +536,8 @@ static u64 nvgpu_mem_linux_sgl_gpu_addr(struct gk20a *g, void *sgl, | |||
536 | static bool nvgpu_mem_linux_sgt_iommuable(struct gk20a *g, | 536 | static bool nvgpu_mem_linux_sgt_iommuable(struct gk20a *g, |
537 | struct nvgpu_sgt *sgt) | 537 | struct nvgpu_sgt *sgt) |
538 | { | 538 | { |
539 | if (nvgpu_is_enabled(g, NVGPU_MM_USE_PHYSICAL_SG)) | ||
540 | return false; | ||
539 | return true; | 541 | return true; |
540 | } | 542 | } |
541 | 543 | ||
diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c index 6cf74048..61dc8bc7 100644 --- a/drivers/gpu/nvgpu/common/linux/nvlink.c +++ b/drivers/gpu/nvgpu/common/linux/nvlink.c | |||
@@ -40,7 +40,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) | |||
40 | 40 | ||
41 | /* Lets find the detected link */ | 41 | /* Lets find the detected link */ |
42 | if (g->nvlink.initialized_links) | 42 | if (g->nvlink.initialized_links) |
43 | link_id = fls(g->nvlink.initialized_links); | 43 | link_id = ffs(g->nvlink.initialized_links) - 1; |
44 | else | 44 | else |
45 | return NVLINK_MAX_LINKS_SW; | 45 | return NVLINK_MAX_LINKS_SW; |
46 | 46 | ||
@@ -49,6 +49,7 @@ static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev) | |||
49 | 49 | ||
50 | return NVLINK_MAX_LINKS_SW; | 50 | return NVLINK_MAX_LINKS_SW; |
51 | } | 51 | } |
52 | |||
52 | static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) | 53 | static int nvgpu_nvlink_early_init(struct nvlink_device *ndev) |
53 | { | 54 | { |
54 | struct gk20a *g = (struct gk20a *) ndev->priv; | 55 | struct gk20a *g = (struct gk20a *) ndev->priv; |
@@ -75,9 +76,10 @@ static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev) | |||
75 | * First check the topology and setup connectivity | 76 | * First check the topology and setup connectivity |
76 | * HACK: we are only enabling one link for now!!! | 77 | * HACK: we are only enabling one link for now!!! |
77 | */ | 78 | */ |
78 | link_id = fls(g->nvlink.discovered_links); | 79 | link_id = ffs(g->nvlink.discovered_links) - 1; |
79 | g->nvlink.links[link_id].remote_info.is_connected = true; | 80 | g->nvlink.links[link_id].remote_info.is_connected = true; |
80 | 81 | g->nvlink.links[link_id].remote_info.device_type = | |
82 | nvgpu_nvlink_endp_tegra; | ||
81 | err = g->ops.nvlink.link_early_init(g, BIT(link_id)); | 83 | err = g->ops.nvlink.link_early_init(g, BIT(link_id)); |
82 | 84 | ||
83 | if (err == 0) { | 85 | if (err == 0) { |
@@ -262,6 +264,7 @@ static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev, | |||
262 | return -EINVAL; | 264 | return -EINVAL; |
263 | 265 | ||
264 | mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); | 266 | mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink); |
267 | |||
265 | switch (mode) { | 268 | switch (mode) { |
266 | case nvgpu_nvlink_sublink_tx_hs: | 269 | case nvgpu_nvlink_sublink_tx_hs: |
267 | return NVLINK_TX_HS; | 270 | return NVLINK_TX_HS; |
@@ -386,9 +389,9 @@ static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev, | |||
386 | } | 389 | } |
387 | #endif | 390 | #endif |
388 | 391 | ||
389 | u32 nvgpu_nvlink_enumerate(struct gk20a *g) | 392 | int nvgpu_nvlink_enumerate(struct gk20a *g) |
390 | { | 393 | { |
391 | u32 err = -ENODEV; | 394 | int err = -ENODEV; |
392 | #ifdef CONFIG_TEGRA_NVLINK | 395 | #ifdef CONFIG_TEGRA_NVLINK |
393 | struct nvlink_device *ndev; | 396 | struct nvlink_device *ndev; |
394 | 397 | ||
@@ -400,9 +403,9 @@ u32 nvgpu_nvlink_enumerate(struct gk20a *g) | |||
400 | return err; | 403 | return err; |
401 | } | 404 | } |
402 | 405 | ||
403 | u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) | 406 | int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) |
404 | { | 407 | { |
405 | u32 err = -ENODEV; | 408 | int err = -ENODEV; |
406 | #ifdef CONFIG_TEGRA_NVLINK | 409 | #ifdef CONFIG_TEGRA_NVLINK |
407 | struct nvlink_device *ndev; | 410 | struct nvlink_device *ndev; |
408 | 411 | ||
@@ -423,10 +426,10 @@ u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off) | |||
423 | return err; | 426 | return err; |
424 | } | 427 | } |
425 | 428 | ||
426 | u32 nvgpu_nvlink_probe(struct gk20a *g) | 429 | int nvgpu_nvlink_probe(struct gk20a *g) |
427 | { | 430 | { |
428 | #ifdef CONFIG_TEGRA_NVLINK | 431 | #ifdef CONFIG_TEGRA_NVLINK |
429 | u32 err = 0; | 432 | int err = 0; |
430 | struct device_node *np = nvgpu_get_node(g); | 433 | struct device_node *np = nvgpu_get_node(g); |
431 | struct device_node *nvlink_np = NULL, *endp_np = NULL; | 434 | struct device_node *nvlink_np = NULL, *endp_np = NULL; |
432 | struct nvlink_device *ndev; | 435 | struct nvlink_device *ndev; |
@@ -435,13 +438,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g) | |||
435 | /* Parse DT */ | 438 | /* Parse DT */ |
436 | if (np) { | 439 | if (np) { |
437 | nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); | 440 | nvlink_np = of_get_child_by_name(np, "nvidia,nvlink"); |
438 | if (nvlink_np) | 441 | if (nvlink_np) { |
439 | endp_np = of_get_child_by_name(np, "endpoint"); | 442 | endp_np = of_get_child_by_name(nvlink_np, "endpoint"); |
443 | } | ||
440 | } | 444 | } |
441 | 445 | ||
442 | if (!endp_np) { | 446 | if (!endp_np) { |
443 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink, | 447 | nvgpu_info(g, "no nvlink DT detected"); |
444 | "No Nvlink DT detected"); | ||
445 | return -ENODEV; | 448 | return -ENODEV; |
446 | } | 449 | } |
447 | 450 | ||
@@ -466,13 +469,13 @@ u32 nvgpu_nvlink_probe(struct gk20a *g) | |||
466 | of_property_read_u32(endp_np, "physical_link", | 469 | of_property_read_u32(endp_np, "physical_link", |
467 | &phys_link_id); | 470 | &phys_link_id); |
468 | 471 | ||
469 | g->nvlink.topology_connected_links = BIT(phys_link_id); | 472 | g->nvlink.connected_links = BIT(phys_link_id); |
470 | 473 | ||
471 | /* Check that we are in dGPU mode */ | 474 | /* Check that we are in dGPU mode */ |
472 | if (ndev->device_id != NVLINK_ENDPT_GV100) { | 475 | if (ndev->device_id != NVLINK_ENDPT_GV100) { |
473 | nvgpu_err(g, "Local nvlink device is not dGPU"); | 476 | nvgpu_err(g, "Local nvlink device is not dGPU"); |
474 | err = -EINVAL; | 477 | err = -EINVAL; |
475 | goto free_nvlink; | 478 | goto free_ndev; |
476 | } | 479 | } |
477 | 480 | ||
478 | /* Fill in device struct */ | 481 | /* Fill in device struct */ |
@@ -500,22 +503,27 @@ u32 nvgpu_nvlink_probe(struct gk20a *g) | |||
500 | err = nvlink_register_device(ndev); | 503 | err = nvlink_register_device(ndev); |
501 | if (err) { | 504 | if (err) { |
502 | nvgpu_err(g, "failed on nvlink device registration"); | 505 | nvgpu_err(g, "failed on nvlink device registration"); |
503 | goto free_nvlink; | 506 | goto free_ndev; |
504 | } | 507 | } |
505 | 508 | ||
506 | /* Register link with core driver */ | 509 | /* Register link with core driver */ |
507 | err = nvlink_register_link(&ndev->link); | 510 | err = nvlink_register_link(&ndev->link); |
508 | if (err) { | 511 | if (err) { |
509 | nvgpu_err(g, "failed on nvlink link registration"); | 512 | nvgpu_err(g, "failed on nvlink link registration"); |
510 | goto free_nvlink; | 513 | goto unregister_ndev; |
511 | } | 514 | } |
512 | 515 | ||
513 | /* Enable NVLINK support */ | 516 | /* Enable NVLINK support */ |
514 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); | 517 | __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true); |
515 | free_nvlink: | 518 | return 0; |
519 | |||
520 | unregister_ndev: | ||
521 | nvlink_unregister_device(ndev); | ||
522 | |||
523 | free_ndev: | ||
516 | nvgpu_kfree(g, ndev); | 524 | nvgpu_kfree(g, ndev); |
525 | g->nvlink.priv = NULL; | ||
517 | return err; | 526 | return err; |
518 | |||
519 | #else | 527 | #else |
520 | return -ENODEV; | 528 | return -ENODEV; |
521 | #endif | 529 | #endif |
diff --git a/drivers/gpu/nvgpu/gk20a/gk20a.h b/drivers/gpu/nvgpu/gk20a/gk20a.h index 02c7d0d9..40656edd 100644 --- a/drivers/gpu/nvgpu/gk20a/gk20a.h +++ b/drivers/gpu/nvgpu/gk20a/gk20a.h | |||
@@ -967,6 +967,8 @@ struct gpu_ops { | |||
967 | void (*isr_stall)(struct gk20a *g); | 967 | void (*isr_stall)(struct gk20a *g); |
968 | bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); | 968 | bool (*is_intr_hub_pending)(struct gk20a *g, u32 mc_intr); |
969 | bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); | 969 | bool (*is_intr_nvlink_pending)(struct gk20a *g, u32 mc_intr); |
970 | bool (*is_stall_and_eng_intr_pending)(struct gk20a *g, | ||
971 | u32 act_eng_id); | ||
970 | u32 (*intr_stall)(struct gk20a *g); | 972 | u32 (*intr_stall)(struct gk20a *g); |
971 | void (*intr_stall_pause)(struct gk20a *g); | 973 | void (*intr_stall_pause)(struct gk20a *g); |
972 | void (*intr_stall_resume)(struct gk20a *g); | 974 | void (*intr_stall_resume)(struct gk20a *g); |
@@ -1064,10 +1066,10 @@ struct gpu_ops { | |||
1064 | int (*check_priv_security)(struct gk20a *g); | 1066 | int (*check_priv_security)(struct gk20a *g); |
1065 | } fuse; | 1067 | } fuse; |
1066 | struct { | 1068 | struct { |
1067 | u32 (*init)(struct gk20a *g); | 1069 | int (*init)(struct gk20a *g); |
1068 | u32 (*discover_ioctrl)(struct gk20a *g); | 1070 | int (*discover_ioctrl)(struct gk20a *g); |
1069 | u32 (*discover_link)(struct gk20a *g); | 1071 | int (*discover_link)(struct gk20a *g); |
1070 | u32 (*isr)(struct gk20a *g); | 1072 | int (*isr)(struct gk20a *g); |
1071 | /* API */ | 1073 | /* API */ |
1072 | int (*link_early_init)(struct gk20a *g, unsigned long mask); | 1074 | int (*link_early_init)(struct gk20a *g, unsigned long mask); |
1073 | u32 (*link_get_mode)(struct gk20a *g, u32 link_id); | 1075 | u32 (*link_get_mode)(struct gk20a *g, u32 link_id); |
diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.c b/drivers/gpu/nvgpu/gv100/flcn_gv100.c new file mode 100644 index 00000000..5820d6c9 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.c | |||
@@ -0,0 +1,55 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | #include <nvgpu/falcon.h> | ||
23 | #include <nvgpu/pmu.h> | ||
24 | |||
25 | #include "gk20a/gk20a.h" | ||
26 | #include "gk20a/flcn_gk20a.h" | ||
27 | #include "gp106/flcn_gp106.h" | ||
28 | #include "gv100/flcn_gv100.h" | ||
29 | |||
30 | #include <nvgpu/hw/gv100/hw_falcon_gv100.h> | ||
31 | |||
32 | void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn) | ||
33 | { | ||
34 | struct gk20a *g = flcn->g; | ||
35 | |||
36 | switch (flcn->flcn_id) { | ||
37 | case FALCON_ID_MINION: | ||
38 | flcn->flcn_base = g->nvlink.minion_base; | ||
39 | flcn->is_falcon_supported = true; | ||
40 | flcn->is_interrupt_enabled = true; | ||
41 | break; | ||
42 | default: | ||
43 | break; | ||
44 | } | ||
45 | |||
46 | if (flcn->is_falcon_supported) { | ||
47 | nvgpu_mutex_init(&flcn->copy_lock); | ||
48 | gk20a_falcon_ops(flcn); | ||
49 | } else { | ||
50 | /* | ||
51 | * Fall back | ||
52 | */ | ||
53 | gp106_falcon_hal_sw_init(flcn); | ||
54 | } | ||
55 | } | ||
diff --git a/drivers/gpu/nvgpu/gv100/flcn_gv100.h b/drivers/gpu/nvgpu/gv100/flcn_gv100.h new file mode 100644 index 00000000..869e74c8 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/flcn_gv100.h | |||
@@ -0,0 +1,27 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | #ifndef __FLCN_GV100_H__ | ||
23 | #define __FLCN_GV100_H__ | ||
24 | |||
25 | void gv100_falcon_hal_sw_init(struct nvgpu_falcon *flcn); | ||
26 | |||
27 | #endif /* __FLCN_GV100_H__ */ | ||
diff --git a/drivers/gpu/nvgpu/gv100/hal_gv100.c b/drivers/gpu/nvgpu/gv100/hal_gv100.c index 6f4ab875..53d61bfb 100644 --- a/drivers/gpu/nvgpu/gv100/hal_gv100.c +++ b/drivers/gpu/nvgpu/gv100/hal_gv100.c | |||
@@ -51,20 +51,19 @@ | |||
51 | #include "gm20b/pmu_gm20b.h" | 51 | #include "gm20b/pmu_gm20b.h" |
52 | #include "gm20b/acr_gm20b.h" | 52 | #include "gm20b/acr_gm20b.h" |
53 | 53 | ||
54 | #include "gp10b/fb_gp10b.h" | ||
55 | #include "gp10b/gr_gp10b.h" | ||
56 | |||
57 | #include "gp106/clk_gp106.h" | 54 | #include "gp106/clk_gp106.h" |
58 | #include "gp106/clk_arb_gp106.h" | 55 | #include "gp106/clk_arb_gp106.h" |
59 | #include "gp106/pmu_gp106.h" | 56 | #include "gp106/pmu_gp106.h" |
60 | #include "gp106/acr_gp106.h" | 57 | #include "gp106/acr_gp106.h" |
61 | #include "gp106/sec2_gp106.h" | 58 | #include "gp106/sec2_gp106.h" |
62 | #include "gp106/bios_gp106.h" | 59 | #include "gp106/bios_gp106.h" |
63 | #include "gv100/bios_gv100.h" | ||
64 | #include "gp106/therm_gp106.h" | 60 | #include "gp106/therm_gp106.h" |
65 | #include "gp106/xve_gp106.h" | 61 | #include "gp106/xve_gp106.h" |
66 | #include "gp106/clk_gp106.h" | 62 | #include "gp106/clk_gp106.h" |
67 | #include "gp106/flcn_gp106.h" | 63 | #include "gp106/flcn_gp106.h" |
64 | |||
65 | #include "gp10b/fb_gp10b.h" | ||
66 | #include "gp10b/gr_gp10b.h" | ||
68 | #include "gp10b/ltc_gp10b.h" | 67 | #include "gp10b/ltc_gp10b.h" |
69 | #include "gp10b/therm_gp10b.h" | 68 | #include "gp10b/therm_gp10b.h" |
70 | #include "gp10b/mc_gp10b.h" | 69 | #include "gp10b/mc_gp10b.h" |
@@ -78,32 +77,33 @@ | |||
78 | #include "gv11b/css_gr_gv11b.h" | 77 | #include "gv11b/css_gr_gv11b.h" |
79 | #include "gv11b/dbg_gpu_gv11b.h" | 78 | #include "gv11b/dbg_gpu_gv11b.h" |
80 | #include "gv11b/hal_gv11b.h" | 79 | #include "gv11b/hal_gv11b.h" |
81 | #include "gv100/gr_gv100.h" | ||
82 | #include "gv11b/gr_gv11b.h" | 80 | #include "gv11b/gr_gv11b.h" |
83 | #include "gv11b/mc_gv11b.h" | 81 | #include "gv11b/mc_gv11b.h" |
84 | #include "gv11b/ltc_gv11b.h" | 82 | #include "gv11b/ltc_gv11b.h" |
85 | #include "gv11b/gv11b.h" | 83 | #include "gv11b/gv11b.h" |
86 | #include "gv11b/ce_gv11b.h" | 84 | #include "gv11b/ce_gv11b.h" |
87 | #include "gv100/gr_ctx_gv100.h" | ||
88 | #include "gv11b/mm_gv11b.h" | 85 | #include "gv11b/mm_gv11b.h" |
89 | #include "gv11b/pmu_gv11b.h" | 86 | #include "gv11b/pmu_gv11b.h" |
90 | #include "gv11b/fb_gv11b.h" | 87 | #include "gv11b/fb_gv11b.h" |
91 | #include "gv100/mm_gv100.h" | ||
92 | #include "gv11b/pmu_gv11b.h" | 88 | #include "gv11b/pmu_gv11b.h" |
93 | #include "gv100/fb_gv100.h" | ||
94 | #include "gv100/fifo_gv100.h" | ||
95 | #include "gv11b/fifo_gv11b.h" | 89 | #include "gv11b/fifo_gv11b.h" |
96 | #include "gv11b/regops_gv11b.h" | 90 | #include "gv11b/regops_gv11b.h" |
97 | |||
98 | #include "gv11b/gv11b_gating_reglist.h" | 91 | #include "gv11b/gv11b_gating_reglist.h" |
99 | #include "gv100/regops_gv100.h" | ||
100 | #include "gv11b/subctx_gv11b.h" | 92 | #include "gv11b/subctx_gv11b.h" |
101 | 93 | ||
102 | #include "gv100.h" | 94 | #include "gv100.h" |
103 | #include "hal_gv100.h" | 95 | #include "hal_gv100.h" |
96 | #include "gv100/bios_gv100.h" | ||
104 | #include "gv100/fb_gv100.h" | 97 | #include "gv100/fb_gv100.h" |
98 | #include "gv100/fifo_gv100.h" | ||
99 | #include "gv100/flcn_gv100.h" | ||
100 | #include "gv100/gr_ctx_gv100.h" | ||
101 | #include "gv100/gr_gv100.h" | ||
102 | #include "gv100/mc_gv100.h" | ||
105 | #include "gv100/mm_gv100.h" | 103 | #include "gv100/mm_gv100.h" |
106 | #include "gv100/pmu_gv100.h" | 104 | #include "gv100/pmu_gv100.h" |
105 | #include "gv100/nvlink_gv100.h" | ||
106 | #include "gv100/regops_gv100.h" | ||
107 | 107 | ||
108 | #include <nvgpu/bus.h> | 108 | #include <nvgpu/bus.h> |
109 | #include <nvgpu/debug.h> | 109 | #include <nvgpu/debug.h> |
@@ -651,7 +651,7 @@ static const struct gpu_ops gv100_ops = { | |||
651 | .apply_smpc_war = gv100_apply_smpc_war, | 651 | .apply_smpc_war = gv100_apply_smpc_war, |
652 | }, | 652 | }, |
653 | .mc = { | 653 | .mc = { |
654 | .intr_enable = mc_gv11b_intr_enable, | 654 | .intr_enable = mc_gv100_intr_enable, |
655 | .intr_unit_config = mc_gp10b_intr_unit_config, | 655 | .intr_unit_config = mc_gp10b_intr_unit_config, |
656 | .isr_stall = mc_gp10b_isr_stall, | 656 | .isr_stall = mc_gp10b_isr_stall, |
657 | .intr_stall = mc_gp10b_intr_stall, | 657 | .intr_stall = mc_gp10b_intr_stall, |
@@ -666,6 +666,9 @@ static const struct gpu_ops gv100_ops = { | |||
666 | .boot_0 = gk20a_mc_boot_0, | 666 | .boot_0 = gk20a_mc_boot_0, |
667 | .is_intr1_pending = mc_gp10b_is_intr1_pending, | 667 | .is_intr1_pending = mc_gp10b_is_intr1_pending, |
668 | .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, | 668 | .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, |
669 | .is_intr_nvlink_pending = gv100_mc_is_intr_nvlink_pending, | ||
670 | .is_stall_and_eng_intr_pending = | ||
671 | gv100_mc_is_stall_and_eng_intr_pending, | ||
669 | }, | 672 | }, |
670 | .debug = { | 673 | .debug = { |
671 | .show_dump = gk20a_debug_show_dump, | 674 | .show_dump = gk20a_debug_show_dump, |
@@ -712,11 +715,30 @@ static const struct gpu_ops gv100_ops = { | |||
712 | .disable_shadow_rom = xve_disable_shadow_rom_gp106, | 715 | .disable_shadow_rom = xve_disable_shadow_rom_gp106, |
713 | }, | 716 | }, |
714 | .falcon = { | 717 | .falcon = { |
715 | .falcon_hal_sw_init = gp106_falcon_hal_sw_init, | 718 | .falcon_hal_sw_init = gv100_falcon_hal_sw_init, |
716 | }, | 719 | }, |
717 | .priv_ring = { | 720 | .priv_ring = { |
718 | .isr = gp10b_priv_ring_isr, | 721 | .isr = gp10b_priv_ring_isr, |
719 | }, | 722 | }, |
723 | .nvlink = { | ||
724 | .discover_ioctrl = gv100_nvlink_discover_ioctrl, | ||
725 | .discover_link = gv100_nvlink_discover_link, | ||
726 | .init = gv100_nvlink_init, | ||
727 | .isr = gv100_nvlink_isr, | ||
728 | /* API */ | ||
729 | .link_early_init = gv100_nvlink_link_early_init, | ||
730 | .link_get_state = gv100_nvlink_link_get_state, | ||
731 | .link_set_mode = gv100_nvlink_link_set_mode, | ||
732 | .link_get_mode = gv100_nvlink_link_get_mode, | ||
733 | .get_sublink_mode = gv100_nvlink_link_get_sublink_mode, | ||
734 | .get_tx_sublink_state = gv100_nvlink_link_get_tx_sublink_state, | ||
735 | .get_rx_sublink_state = gv100_nvlink_link_get_rx_sublink_state, | ||
736 | .set_sublink_mode = gv100_nvlink_link_set_sublink_mode, | ||
737 | .interface_init = gv100_nvlink_interface_init, | ||
738 | .reg_init = gv100_nvlink_reg_init, | ||
739 | .shutdown = gv100_nvlink_shutdown, | ||
740 | .early_init = gv100_nvlink_early_init, | ||
741 | }, | ||
720 | .chip_init_gpu_characteristics = gv100_init_gpu_characteristics, | 742 | .chip_init_gpu_characteristics = gv100_init_gpu_characteristics, |
721 | .get_litter_value = gv100_get_litter_value, | 743 | .get_litter_value = gv100_get_litter_value, |
722 | }; | 744 | }; |
@@ -751,6 +773,7 @@ int gv100_init_hal(struct gk20a *g) | |||
751 | gops->xve = gv100_ops.xve; | 773 | gops->xve = gv100_ops.xve; |
752 | gops->falcon = gv100_ops.falcon; | 774 | gops->falcon = gv100_ops.falcon; |
753 | gops->priv_ring = gv100_ops.priv_ring; | 775 | gops->priv_ring = gv100_ops.priv_ring; |
776 | gops->nvlink = gv100_ops.nvlink; | ||
754 | 777 | ||
755 | /* clocks */ | 778 | /* clocks */ |
756 | gops->clk.init_clk_support = gv100_ops.clk.init_clk_support; | 779 | gops->clk.init_clk_support = gv100_ops.clk.init_clk_support; |
diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.c b/drivers/gpu/nvgpu/gv100/mc_gv100.c new file mode 100644 index 00000000..a3f8ab06 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/mc_gv100.c | |||
@@ -0,0 +1,94 @@ | |||
1 | /* | ||
2 | * GV100 master | ||
3 | * | ||
4 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
5 | * | ||
6 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
7 | * copy of this software and associated documentation files (the "Software"), | ||
8 | * to deal in the Software without restriction, including without limitation | ||
9 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
10 | * and/or sell copies of the Software, and to permit persons to whom the | ||
11 | * Software is furnished to do so, subject to the following conditions: | ||
12 | * | ||
13 | * The above copyright notice and this permission notice shall be included in | ||
14 | * all copies or substantial portions of the Software. | ||
15 | * | ||
16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
21 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
22 | * DEALINGS IN THE SOFTWARE. | ||
23 | */ | ||
24 | |||
25 | #include <linux/types.h> | ||
26 | |||
27 | #include "gk20a/gk20a.h" | ||
28 | |||
29 | #include "gp10b/mc_gp10b.h" | ||
30 | |||
31 | #include "mc_gv100.h" | ||
32 | #include "gv11b/fb_gv11b.h" | ||
33 | |||
34 | #include <nvgpu/hw/gv100/hw_mc_gv100.h> | ||
35 | |||
36 | void mc_gv100_intr_enable(struct gk20a *g) | ||
37 | { | ||
38 | u32 eng_intr_mask = gk20a_fifo_engine_interrupt_mask(g); | ||
39 | |||
40 | gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_STALLING), | ||
41 | 0xffffffff); | ||
42 | gk20a_writel(g, mc_intr_en_clear_r(NVGPU_MC_INTR_NONSTALLING), | ||
43 | 0xffffffff); | ||
44 | gv11b_fb_disable_hub_intr(g, STALL_REG_INDEX, HUB_INTR_TYPE_ALL); | ||
45 | |||
46 | g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING] = | ||
47 | mc_intr_pfifo_pending_f() | | ||
48 | mc_intr_hub_pending_f() | | ||
49 | mc_intr_priv_ring_pending_f() | | ||
50 | mc_intr_pbus_pending_f() | | ||
51 | mc_intr_ltc_pending_f() | | ||
52 | mc_intr_nvlink_pending_f() | | ||
53 | eng_intr_mask; | ||
54 | |||
55 | g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING] = | ||
56 | mc_intr_pfifo_pending_f() | ||
57 | | eng_intr_mask; | ||
58 | |||
59 | /* TODO: Enable PRI faults for HUB ECC err intr */ | ||
60 | gv11b_fb_enable_hub_intr(g, STALL_REG_INDEX, g->mm.hub_intr_types); | ||
61 | |||
62 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_STALLING), | ||
63 | g->mc_intr_mask_restore[NVGPU_MC_INTR_STALLING]); | ||
64 | |||
65 | gk20a_writel(g, mc_intr_en_set_r(NVGPU_MC_INTR_NONSTALLING), | ||
66 | g->mc_intr_mask_restore[NVGPU_MC_INTR_NONSTALLING]); | ||
67 | |||
68 | } | ||
69 | |||
70 | bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0) | ||
71 | { | ||
72 | return ((mc_intr_0 & mc_intr_nvlink_pending_f()) ? true : false); | ||
73 | } | ||
74 | |||
75 | bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id) | ||
76 | { | ||
77 | u32 mc_intr_0 = gk20a_readl(g, mc_intr_r(0)); | ||
78 | u32 stall_intr, eng_intr_mask; | ||
79 | |||
80 | eng_intr_mask = gk20a_fifo_act_eng_interrupt_mask(g, act_eng_id); | ||
81 | if (mc_intr_0 & eng_intr_mask) | ||
82 | return true; | ||
83 | |||
84 | stall_intr = mc_intr_pfifo_pending_f() | | ||
85 | mc_intr_hub_pending_f() | | ||
86 | mc_intr_priv_ring_pending_f() | | ||
87 | mc_intr_pbus_pending_f() | | ||
88 | mc_intr_ltc_pending_f() | | ||
89 | mc_intr_nvlink_pending_f(); | ||
90 | if (mc_intr_0 & stall_intr) | ||
91 | return true; | ||
92 | |||
93 | return false; | ||
94 | } | ||
diff --git a/drivers/gpu/nvgpu/gv100/mc_gv100.h b/drivers/gpu/nvgpu/gv100/mc_gv100.h new file mode 100644 index 00000000..4aff4a36 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/mc_gv100.h | |||
@@ -0,0 +1,30 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef MC_GV100_H | ||
24 | #define MC_GV100_H | ||
25 | struct gk20a; | ||
26 | |||
27 | void mc_gv100_intr_enable(struct gk20a *g); | ||
28 | bool gv100_mc_is_intr_nvlink_pending(struct gk20a *g, u32 mc_intr_0); | ||
29 | bool gv100_mc_is_stall_and_eng_intr_pending(struct gk20a *g, u32 act_eng_id); | ||
30 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.c b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c new file mode 100644 index 00000000..e5e65c51 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.c | |||
@@ -0,0 +1,2654 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #include <nvgpu/nvgpu_common.h> | ||
24 | #include <nvgpu/bios.h> | ||
25 | #include <nvgpu/firmware.h> | ||
26 | #include <nvgpu/bitops.h> | ||
27 | #include <nvgpu/nvlink.h> | ||
28 | #include <nvgpu/enabled.h> | ||
29 | |||
30 | #include "gk20a/gk20a.h" | ||
31 | #include "nvlink_gv100.h" | ||
32 | |||
33 | #include <nvgpu/hw/gv100/hw_falcon_gv100.h> | ||
34 | #include <nvgpu/hw/gv100/hw_top_gv100.h> | ||
35 | #include <nvgpu/hw/gv100/hw_nvlinkip_discovery_gv100.h> | ||
36 | #include <nvgpu/hw/gv100/hw_nvlipt_gv100.h> | ||
37 | #include <nvgpu/hw/gv100/hw_ioctrl_gv100.h> | ||
38 | #include <nvgpu/hw/gv100/hw_minion_gv100.h> | ||
39 | #include <nvgpu/hw/gv100/hw_nvl_gv100.h> | ||
40 | #include <nvgpu/hw/gv100/hw_ioctrlmif_gv100.h> | ||
41 | #include <nvgpu/hw/gv100/hw_trim_gv100.h> | ||
42 | #include <nvgpu/hw/gv100/hw_nvtlc_gv100.h> | ||
43 | |||
44 | #include <nvgpu/hw/gv100/hw_fb_gv100.h> | ||
45 | #include <nvgpu/hw/gv100/hw_mc_gv100.h> | ||
46 | |||
47 | #define NVLINK_PLL_ON_TIMEOUT_MS 30 | ||
48 | #define NVLINK_SUBLINK_TIMEOUT_MS 200 | ||
49 | /* | ||
50 | * The manuals are missing some useful defines | ||
51 | * we add them for now | ||
52 | */ | ||
53 | |||
54 | #define IPT_INTR_CONTROL_LINK(i) (nvlipt_intr_control_link0_r() + (i)*4) | ||
55 | #define IPT_ERR_UC_STATUS_LINK(i) (nvlipt_err_uc_status_link0_r() + (i)*36) | ||
56 | #define IPT_ERR_UC_MASK_LINK(i) (nvlipt_err_uc_mask_link0_r() + (i)*36) | ||
57 | #define IPT_ERR_UC_SEVERITY_LINK(i) (nvlipt_err_uc_severity_link0_r() + (i)*36) | ||
58 | #define IPT_ERR_UC_FIRST_LINK(i) (nvlipt_err_uc_first_link0_r() + (i)*36) | ||
59 | #define IPT_ERR_UC_ADVISORY_LINK(i) (nvlipt_err_uc_advisory_link0_r() + (i)*36) | ||
60 | #define IPT_ERR_C_STATUS_LINK(i) (nvlipt_err_c_status_link0_r() + (i)*36) | ||
61 | #define IPT_ERR_C_MASK_LINK(i) (nvlipt_err_c_mask_link0_r() + (i)*36) | ||
62 | #define IPT_ERR_C_FIRST_LINK(i) (nvlipt_err_c_first_link0_r() + (i)*36) | ||
63 | #define IPT_ERR_CONTROL_LINK(i) (nvlipt_err_control_link0_r() + (i)*4) | ||
64 | |||
65 | #define IPT_ERR_UC_ACTIVE_BITS (nvlipt_err_uc_status_link0_dlprotocol_f(1) | \ | ||
66 | nvlipt_err_uc_status_link0_datapoisoned_f(1) | \ | ||
67 | nvlipt_err_uc_status_link0_flowcontrol_f(1) | \ | ||
68 | nvlipt_err_uc_status_link0_responsetimeout_f(1) | \ | ||
69 | nvlipt_err_uc_status_link0_targeterror_f(1) | \ | ||
70 | nvlipt_err_uc_status_link0_unexpectedresponse_f(1) | \ | ||
71 | nvlipt_err_uc_status_link0_receiveroverflow_f(1) | \ | ||
72 | nvlipt_err_uc_status_link0_malformedpacket_f(1) | \ | ||
73 | nvlipt_err_uc_status_link0_stompedpacketreceived_f(1) | \ | ||
74 | nvlipt_err_uc_status_link0_unsupportedrequest_f(1) | \ | ||
75 | nvlipt_err_uc_status_link0_ucinternal_f(1)) | ||
76 | |||
77 | |||
78 | #define MINION_FALCON_INTR_MASK (minion_falcon_irqmset_wdtmr_set_f() | \ | ||
79 | minion_falcon_irqmset_halt_set_f() | \ | ||
80 | minion_falcon_irqmset_exterr_set_f()| \ | ||
81 | minion_falcon_irqmset_swgen0_set_f()| \ | ||
82 | minion_falcon_irqmset_swgen1_set_f()) | ||
83 | |||
84 | #define MINION_FALCON_INTR_DEST ( \ | ||
85 | minion_falcon_irqdest_host_wdtmr_host_f() | \ | ||
86 | minion_falcon_irqdest_host_halt_host_f() | \ | ||
87 | minion_falcon_irqdest_host_exterr_host_f() | \ | ||
88 | minion_falcon_irqdest_host_swgen0_host_f() | \ | ||
89 | minion_falcon_irqdest_host_swgen1_host_f() | \ | ||
90 | minion_falcon_irqdest_target_wdtmr_host_normal_f() | \ | ||
91 | minion_falcon_irqdest_target_halt_host_normal_f() | \ | ||
92 | minion_falcon_irqdest_target_exterr_host_normal_f() | \ | ||
93 | minion_falcon_irqdest_target_swgen0_host_normal_f() | \ | ||
94 | minion_falcon_irqdest_target_swgen1_host_normal_f()) | ||
95 | |||
96 | /* Minion FW header format */ | ||
97 | union gv100_minion_hdr { | ||
98 | struct { | ||
99 | u32 os_code_offset; | ||
100 | u32 os_code_size; | ||
101 | u32 os_data_offset; | ||
102 | u32 os_data_size; | ||
103 | u32 num_apps; | ||
104 | u32 app_0_code_start; | ||
105 | }; | ||
106 | u8 raw_data[256]; | ||
107 | }; | ||
108 | |||
109 | struct __nvlink_reginit { | ||
110 | u32 addr; | ||
111 | u32 value; | ||
112 | }; | ||
113 | |||
114 | static const struct __nvlink_reginit __nvlink_reginit_per_link_tegra[] = { | ||
115 | /* NVTLC when connected to Tegra */ | ||
116 | { 0x300U, 0x00800040U }, | ||
117 | { 0x304U, 0x00000020U }, | ||
118 | { 0x308U, 0x00000020U }, | ||
119 | { 0x30CU, 0x00000020U }, | ||
120 | { 0x310U, 0x00000020U }, | ||
121 | { 0x314U, 0x00800040U }, | ||
122 | { 0x318U, 0x00000000U }, | ||
123 | { 0x31CU, 0x00000000U }, | ||
124 | { 0x200U, 0x007F003FU }, | ||
125 | { 0x204U, 0x007F005FU }, | ||
126 | { 0x208U, 0x007F007FU }, | ||
127 | { 0x20CU, 0x007F009FU }, | ||
128 | { 0x210U, 0x007F00BFU }, | ||
129 | { 0x214U, 0x00FF003FU }, | ||
130 | { 0x218U, 0x00FF003FU }, | ||
131 | { 0x21CU, 0x00FF003FU }, | ||
132 | { 0xB00U, 0x010000BEU }, | ||
133 | { 0xB04U, 0x00000064U }, | ||
134 | { 0xB08U, 0x00000000U }, | ||
135 | { 0xB0CU, 0x00000020U }, | ||
136 | { 0xB10U, 0x00000000U }, | ||
137 | { 0xB14U, 0x010000BEU }, | ||
138 | { 0xB18U, 0x00000000U }, | ||
139 | { 0xB1CU, 0x00000000U }, | ||
140 | { 0xA00U, 0x00FF00BDU }, | ||
141 | { 0xA04U, 0x00FF0121U }, | ||
142 | { 0xA08U, 0x00FF0121U }, | ||
143 | { 0xA0CU, 0x00FF0141U }, | ||
144 | { 0xA10U, 0x00FF0141U }, | ||
145 | { 0xA14U, 0x01FF01FFU }, | ||
146 | { 0xA18U, 0x01FF01FFU }, | ||
147 | { 0xA1CU, 0x01FF01FFU }, | ||
148 | { 0xF04U, 0x00FFFFFFU }, | ||
149 | { 0xF0CU, 0x00FFFFFFU }, | ||
150 | { 0xF1CU, 0x003FFFFFU }, | ||
151 | { 0xF24U, 0x003FFFFFU }, | ||
152 | { 0x704U, 0x003FFFFFU }, | ||
153 | { 0x70CU, 0x003FFFFFU }, | ||
154 | { 0x400U, 0x00000001U }, | ||
155 | { 0xC00U, 0x00000001U }, | ||
156 | }; | ||
157 | |||
158 | static const struct __nvlink_reginit __nvlink_reginit_per_link_gpu[] = { | ||
159 | /* NVTLC for PEER GPU */ | ||
160 | { 0x300U, 0x00800040U }, | ||
161 | { 0x304U, 0x00000000U }, | ||
162 | { 0x308U, 0x00000000U }, | ||
163 | { 0x30CU, 0x00000000U }, | ||
164 | { 0x310U, 0x00000000U }, | ||
165 | { 0x314U, 0x00800040U }, | ||
166 | { 0x318U, 0x00000000U }, | ||
167 | { 0x31CU, 0x00000000U }, | ||
168 | { 0x200U, 0x007F003FU }, | ||
169 | { 0x204U, 0x007F003FU }, | ||
170 | { 0x208U, 0x007F003FU }, | ||
171 | { 0x20CU, 0x007F003FU }, | ||
172 | { 0x210U, 0x007F003FU }, | ||
173 | { 0x214U, 0x00FF007FU }, | ||
174 | { 0x218U, 0x00FF007FU }, | ||
175 | { 0x21CU, 0x00FF007FU }, | ||
176 | { 0xB00U, 0x010000C0U }, | ||
177 | { 0xB04U, 0x00000000U }, | ||
178 | { 0xB08U, 0x00000000U }, | ||
179 | { 0xB0CU, 0x00000000U }, | ||
180 | { 0xB10U, 0x00000000U }, | ||
181 | { 0xB14U, 0x010000C0U }, | ||
182 | { 0xB18U, 0x00000000U }, | ||
183 | { 0xB1CU, 0x00000000U }, | ||
184 | { 0xA00U, 0x00FF00BFU }, | ||
185 | { 0xA04U, 0x00FF00BFU }, | ||
186 | { 0xA08U, 0x00FF00BFU }, | ||
187 | { 0xA0CU, 0x00FF00BFU }, | ||
188 | { 0xA10U, 0x00FF00BFU }, | ||
189 | { 0xA14U, 0x01FF017FU }, | ||
190 | { 0xA18U, 0x01FF017FU }, | ||
191 | { 0xA1CU, 0x01FF017FU }, | ||
192 | { 0xF04U, 0x00FFFFFFU }, | ||
193 | { 0xF0CU, 0x00FFFFFFU }, | ||
194 | { 0xF1CU, 0x003FFFFFU }, | ||
195 | { 0xF24U, 0x003FFFFFU }, | ||
196 | { 0x704U, 0x003FFFFFU }, | ||
197 | { 0x70CU, 0x003FFFFFU }, | ||
198 | { 0x400U, 0x00000001U }, | ||
199 | { 0xC00U, 0x00000001U }, | ||
200 | }; | ||
201 | |||
202 | |||
203 | #define NVL_DEVICE(str) nvlinkip_discovery_common_device_##str##_v() | ||
204 | |||
205 | const char * __gv100_device_type_to_str(u32 type) | ||
206 | { | ||
207 | if (type == NVL_DEVICE(ioctrl)) | ||
208 | return "IOCTRL"; | ||
209 | if (type == NVL_DEVICE(dlpl)) | ||
210 | return "DL/PL"; | ||
211 | if (type == NVL_DEVICE(nvltlc)) | ||
212 | return "NVLTLC"; | ||
213 | if (type == NVL_DEVICE(ioctrlmif)) | ||
214 | return "IOCTRLMIF"; | ||
215 | if (type == NVL_DEVICE(nvlipt)) | ||
216 | return "NVLIPT"; | ||
217 | if (type == NVL_DEVICE(minion)) | ||
218 | return "MINION"; | ||
219 | if (type == NVL_DEVICE(dlpl_multicast)) | ||
220 | return "DL/PL MULTICAST"; | ||
221 | if (type == NVL_DEVICE(nvltlc_multicast)) | ||
222 | return "NVLTLC MULTICAST"; | ||
223 | if (type == NVL_DEVICE(ioctrlmif_multicast)) | ||
224 | return "IOCTRLMIF MULTICAST"; | ||
225 | if (type == NVL_DEVICE(nvltlc_multicast)) | ||
226 | return "NVLTLC MULTICAST"; | ||
227 | return "UNKNOWN"; | ||
228 | } | ||
229 | |||
230 | /* | ||
231 | * Function prototypes | ||
232 | */ | ||
233 | static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask); | ||
234 | static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g); | ||
235 | static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask); | ||
236 | static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g, | ||
237 | unsigned long mask, bool sync); | ||
238 | |||
239 | |||
240 | /* | ||
241 | ******************************************************************************* | ||
242 | * IP specific functions * | ||
243 | ******************************************************************************* | ||
244 | */ | ||
245 | |||
246 | /* | ||
247 | *-----------------------------------------------------------------------------* | ||
248 | * MINION API | ||
249 | *-----------------------------------------------------------------------------* | ||
250 | */ | ||
251 | |||
252 | /* | ||
253 | * Initialization of link specific interrupts | ||
254 | */ | ||
255 | static void gv100_nvlink_minion_link_intr_enable(struct gk20a *g, u32 link_id, | ||
256 | bool enable) | ||
257 | { | ||
258 | u32 intr, links; | ||
259 | |||
260 | /* Only stall interrupts for now */ | ||
261 | intr = MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); | ||
262 | links = minion_minion_intr_stall_en_link_v(intr); | ||
263 | |||
264 | if (enable) | ||
265 | links |= BIT(link_id); | ||
266 | else | ||
267 | links &= ~BIT(link_id); | ||
268 | |||
269 | intr = set_field(intr, minion_minion_intr_stall_en_link_m(), | ||
270 | minion_minion_intr_stall_en_link_f(links)); | ||
271 | MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), intr); | ||
272 | } | ||
273 | |||
274 | /* | ||
275 | * Initialization of falcon interrupts | ||
276 | */ | ||
277 | static void gv100_nvlink_minion_falcon_intr_enable(struct gk20a *g, bool enable) | ||
278 | { | ||
279 | u32 reg; | ||
280 | |||
281 | reg = MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); | ||
282 | if (enable) { | ||
283 | reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(), | ||
284 | minion_minion_intr_stall_en_fatal_enable_f()); | ||
285 | reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(), | ||
286 | minion_minion_intr_stall_en_nonfatal_enable_f()); | ||
287 | reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(), | ||
288 | minion_minion_intr_stall_en_falcon_stall_enable_f()); | ||
289 | reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(), | ||
290 | minion_minion_intr_stall_en_falcon_nostall_enable_f()); | ||
291 | } else { | ||
292 | reg = set_field(reg, minion_minion_intr_stall_en_fatal_m(), | ||
293 | minion_minion_intr_stall_en_fatal_disable_f()); | ||
294 | reg = set_field(reg, minion_minion_intr_stall_en_nonfatal_m(), | ||
295 | minion_minion_intr_stall_en_nonfatal_disable_f()); | ||
296 | reg = set_field(reg, minion_minion_intr_stall_en_falcon_stall_m(), | ||
297 | minion_minion_intr_stall_en_falcon_stall_disable_f()); | ||
298 | reg = set_field(reg, minion_minion_intr_stall_en_falcon_nostall_m(), | ||
299 | minion_minion_intr_stall_en_falcon_nostall_disable_f()); | ||
300 | } | ||
301 | |||
302 | MINION_REG_WR32(g, minion_minion_intr_stall_en_r(), reg); | ||
303 | } | ||
304 | |||
305 | /* | ||
306 | * Initialize minion IP interrupts | ||
307 | */ | ||
308 | static void gv100_nvlink_initialize_minion(struct gk20a *g) | ||
309 | { | ||
310 | /* Disable non-stall tree */ | ||
311 | MINION_REG_WR32(g, minion_minion_intr_nonstall_en_r(), 0x0); | ||
312 | |||
313 | gv100_nvlink_minion_falcon_intr_enable(g, true); | ||
314 | } | ||
315 | |||
316 | /* | ||
317 | * Check if minion is up | ||
318 | */ | ||
319 | static bool __gv100_nvlink_minion_is_running(struct gk20a *g) | ||
320 | { | ||
321 | |||
322 | /* if minion is booted and not halted, it is running */ | ||
323 | if ((MINION_REG_RD32(g, minion_minion_status_r()) & | ||
324 | minion_minion_status_status_f(1)) && | ||
325 | (!minion_falcon_irqstat_halt_v( | ||
326 | MINION_REG_RD32(g, minion_falcon_irqstat_r())))) | ||
327 | return true; | ||
328 | |||
329 | return false; | ||
330 | } | ||
331 | |||
332 | /* | ||
333 | * Falcon specific ISR handling | ||
334 | */ | ||
335 | static bool gv100_nvlink_minion_falcon_isr(struct gk20a *g) | ||
336 | { | ||
337 | u32 intr; | ||
338 | |||
339 | intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) & | ||
340 | MINION_REG_RD32(g, minion_falcon_irqmask_r()); | ||
341 | |||
342 | if (!intr) | ||
343 | return true; | ||
344 | |||
345 | if (intr & falcon_falcon_irqstat_exterr_true_f()) { | ||
346 | nvgpu_err(g, "FALCON EXT ADDR: 0x%x 0x%x 0x%x", | ||
347 | MINION_REG_RD32(g, 0x244), | ||
348 | MINION_REG_RD32(g, 0x248), | ||
349 | MINION_REG_RD32(g, 0x24c)); | ||
350 | } | ||
351 | |||
352 | MINION_REG_WR32(g, minion_falcon_irqsclr_r(), intr); | ||
353 | |||
354 | nvgpu_err(g, "FATAL minion IRQ: 0x%08x", intr); | ||
355 | |||
356 | |||
357 | intr = MINION_REG_RD32(g, minion_falcon_irqstat_r()) & | ||
358 | MINION_REG_RD32(g, minion_falcon_irqmask_r()); | ||
359 | |||
360 | return (intr == 0); | ||
361 | } | ||
362 | |||
363 | /* | ||
364 | * Link Specific ISR | ||
365 | */ | ||
366 | |||
367 | static bool gv100_nvlink_minion_link_isr(struct gk20a *g, u32 link_id) | ||
368 | { | ||
369 | u32 intr, code; | ||
370 | bool fatal = false; | ||
371 | |||
372 | intr = MINION_REG_RD32(g, minion_nvlink_link_intr_r(link_id)); | ||
373 | code = minion_nvlink_link_intr_code_v(intr); | ||
374 | |||
375 | if (code == minion_nvlink_link_intr_code_swreq_v()) { | ||
376 | nvgpu_err(g, " Intr SWREQ, link: %d subcode: %x", | ||
377 | link_id, minion_nvlink_link_intr_subcode_v(intr)); | ||
378 | } else if (code == minion_nvlink_link_intr_code_pmdisabled_v()) { | ||
379 | nvgpu_err(g, " Fatal Intr PMDISABLED, link: %d subcode: %x", | ||
380 | link_id, minion_nvlink_link_intr_subcode_v(intr)); | ||
381 | fatal = true; | ||
382 | } else if (code == minion_nvlink_link_intr_code_na_v()) { | ||
383 | nvgpu_err(g, " Fatal Intr NA, link: %d subcode: %x", | ||
384 | link_id, minion_nvlink_link_intr_subcode_v(intr)); | ||
385 | fatal = true; | ||
386 | } else if (code == minion_nvlink_link_intr_code_dlreq_v()) { | ||
387 | nvgpu_err(g, " Fatal Intr DLREQ, link: %d subcode: %x", | ||
388 | link_id, minion_nvlink_link_intr_subcode_v(intr)); | ||
389 | fatal = true; | ||
390 | } else { | ||
391 | nvgpu_err(g, " Fatal Intr UNKN:%x, link: %d subcode: %x", code, | ||
392 | link_id, minion_nvlink_link_intr_subcode_v(intr)); | ||
393 | fatal = true; | ||
394 | } | ||
395 | |||
396 | if (fatal) | ||
397 | gv100_nvlink_minion_link_intr_enable(g, link_id, false); | ||
398 | |||
399 | intr = set_field(intr, minion_nvlink_link_intr_state_m(), | ||
400 | minion_nvlink_link_intr_state_f(1)); | ||
401 | MINION_REG_WR32(g, minion_nvlink_link_intr_r(link_id), intr); | ||
402 | |||
403 | return true; | ||
404 | } | ||
405 | |||
406 | /* | ||
407 | * Global minion routine to service interrupts | ||
408 | */ | ||
409 | static bool gv100_nvlink_minion_isr(struct gk20a *g) { | ||
410 | |||
411 | u32 intr, i; | ||
412 | unsigned long links; | ||
413 | |||
414 | intr = MINION_REG_RD32(g, minion_minion_intr_r()) & | ||
415 | MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); | ||
416 | |||
417 | if (minion_minion_intr_falcon_stall_v(intr) || | ||
418 | minion_minion_intr_falcon_nostall_v(intr)) | ||
419 | gv100_nvlink_minion_falcon_isr(g); | ||
420 | |||
421 | if (minion_minion_intr_fatal_v(intr)) { | ||
422 | gv100_nvlink_minion_falcon_intr_enable(g, false); | ||
423 | MINION_REG_WR32(g, minion_minion_intr_r(), | ||
424 | minion_minion_intr_fatal_f(1)); | ||
425 | } | ||
426 | |||
427 | if (minion_minion_intr_nonfatal_v(intr)) | ||
428 | MINION_REG_WR32(g, minion_minion_intr_r(), | ||
429 | minion_minion_intr_nonfatal_f(1)); | ||
430 | |||
431 | links = minion_minion_intr_link_v(intr) & g->nvlink.enabled_links; | ||
432 | |||
433 | if (links) | ||
434 | for_each_set_bit(i, &links, 32) | ||
435 | gv100_nvlink_minion_link_isr(g, i); | ||
436 | |||
437 | /* Re-test interrupt status */ | ||
438 | intr = MINION_REG_RD32(g, minion_minion_intr_r()) & | ||
439 | MINION_REG_RD32(g, minion_minion_intr_stall_en_r()); | ||
440 | |||
441 | return (intr == 0); | ||
442 | } | ||
443 | |||
444 | /* | ||
445 | * Load minion FW and set up bootstrap | ||
446 | */ | ||
447 | static u32 gv100_nvlink_minion_load(struct gk20a *g) | ||
448 | { | ||
449 | struct bin_hdr *hdr = NULL; | ||
450 | struct nvgpu_firmware *minion_fw = NULL; | ||
451 | union gv100_minion_hdr *minion_hdr; | ||
452 | u32 *minion_ucode = NULL; | ||
453 | u32 err = 0; | ||
454 | struct nvgpu_timeout timeout; | ||
455 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
456 | u32 reg; | ||
457 | u32 app; | ||
458 | |||
459 | nvgpu_log_fn(g, " "); | ||
460 | |||
461 | if (__gv100_nvlink_minion_is_running(g)) | ||
462 | return 0; | ||
463 | |||
464 | /* get mem unlock ucode binary */ | ||
465 | minion_fw = nvgpu_request_firmware(g, "minion.bin", 0); | ||
466 | if (!minion_fw) { | ||
467 | nvgpu_err(g, "minion ucode get fail"); | ||
468 | err = -ENOENT; | ||
469 | goto exit; | ||
470 | } | ||
471 | |||
472 | /* nvdec falcon reset */ | ||
473 | nvgpu_flcn_reset(&g->minion_flcn); | ||
474 | |||
475 | hdr = (struct bin_hdr *) minion_fw->data; | ||
476 | |||
477 | minion_hdr = (union gv100_minion_hdr *) (minion_fw->data + | ||
478 | hdr->header_offset); | ||
479 | minion_ucode = (u32 *) (minion_fw->data + hdr->data_offset); | ||
480 | |||
481 | nvgpu_log(g, gpu_dbg_nvlink, | ||
482 | "os_code_offset: 0x%x", minion_hdr->os_code_offset); | ||
483 | nvgpu_log(g, gpu_dbg_nvlink, | ||
484 | "os_code_size: 0x%x", minion_hdr->os_code_size); | ||
485 | nvgpu_log(g, gpu_dbg_nvlink, | ||
486 | "os_data_offset: 0x%x", minion_hdr->os_data_offset); | ||
487 | nvgpu_log(g, gpu_dbg_nvlink, | ||
488 | "os_data_size: 0x%x", minion_hdr->os_data_size); | ||
489 | nvgpu_log(g, gpu_dbg_nvlink, | ||
490 | "num_apps: 0x%x", minion_hdr->num_apps); | ||
491 | nvgpu_log(g, gpu_dbg_nvlink, | ||
492 | "app_0_code_start: 0x%x", minion_hdr->app_0_code_start); | ||
493 | |||
494 | /* Clear interrupts */ | ||
495 | nvgpu_flcn_set_irq(&g->minion_flcn, true, MINION_FALCON_INTR_MASK, | ||
496 | MINION_FALCON_INTR_DEST); | ||
497 | |||
498 | /* Copy Non Secure IMEM code */ | ||
499 | nvgpu_flcn_copy_to_imem(&g->minion_flcn, 0, | ||
500 | (u8 *)&minion_ucode[minion_hdr->os_code_offset >> 2], | ||
501 | minion_hdr->os_code_size, 0, false, | ||
502 | GET_IMEM_TAG(minion_hdr->os_code_offset)); | ||
503 | |||
504 | /* Copy Non Secure DMEM code */ | ||
505 | nvgpu_flcn_copy_to_dmem(&g->minion_flcn, 0, | ||
506 | (u8 *)&minion_ucode[minion_hdr->os_data_offset >> 2], | ||
507 | minion_hdr->os_data_size, 0); | ||
508 | |||
509 | /* If apps are there load them securely */ | ||
510 | if (minion_hdr->num_apps) { | ||
511 | for (app = 0; app < minion_hdr->num_apps; app++) { | ||
512 | u32 app_code_start, app_code_size; | ||
513 | u32 app_data_start, app_data_size; | ||
514 | |||
515 | app_code_start = | ||
516 | *((u32 *) &minion_hdr->app_0_code_start + | ||
517 | 2*app); | ||
518 | app_code_size = | ||
519 | *((u32 *) &minion_hdr->app_0_code_start + | ||
520 | 2*app + 1); | ||
521 | app_data_start = | ||
522 | *((u32 *) &minion_hdr->app_0_code_start + | ||
523 | 2*minion_hdr->num_apps + 2*app); | ||
524 | app_data_size = | ||
525 | *((u32 *) &minion_hdr->app_0_code_start + | ||
526 | 2*minion_hdr->num_apps + 2*app + 1); | ||
527 | |||
528 | nvgpu_log(g, gpu_dbg_nvlink, | ||
529 | " app %d code_offset 0x%x", app, app_code_start); | ||
530 | nvgpu_log(g, gpu_dbg_nvlink, | ||
531 | " app %d code_size 0x%x", app, app_code_size); | ||
532 | nvgpu_log(g, gpu_dbg_nvlink, | ||
533 | " app %d data_offset 0x%x", app, app_data_start); | ||
534 | nvgpu_log(g, gpu_dbg_nvlink, | ||
535 | " app %d data_size 0x%x", app, app_data_size); | ||
536 | |||
537 | if (app_code_size) | ||
538 | nvgpu_flcn_copy_to_imem(&g->minion_flcn, | ||
539 | app_code_start, | ||
540 | (u8 *)&minion_ucode[ | ||
541 | app_code_start >> 2], | ||
542 | app_code_size, 0, true, | ||
543 | GET_IMEM_TAG(app_code_start)); | ||
544 | |||
545 | if (app_data_size) | ||
546 | nvgpu_flcn_copy_to_dmem(&g->minion_flcn, | ||
547 | app_data_start, | ||
548 | (u8 *)&minion_ucode[ | ||
549 | app_data_start >> 2], | ||
550 | app_data_size, 0); | ||
551 | } | ||
552 | } | ||
553 | |||
554 | /* set BOOTVEC to start of non-secure code */ | ||
555 | nvgpu_flcn_bootstrap(&g->minion_flcn, 0x0); | ||
556 | |||
557 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
558 | NVGPU_TIMER_CPU_TIMER); | ||
559 | |||
560 | do { | ||
561 | reg = MINION_REG_RD32(g, minion_minion_status_r()); | ||
562 | |||
563 | if (minion_minion_status_status_v(reg)) { | ||
564 | /* Minion sequence completed, check status */ | ||
565 | if (minion_minion_status_status_v(reg) != | ||
566 | minion_minion_status_status_boot_v()) { | ||
567 | nvgpu_err(g, "MINION init sequence failed: 0x%x", | ||
568 | minion_minion_status_status_v(reg)); | ||
569 | err = -EINVAL; | ||
570 | |||
571 | goto exit; | ||
572 | } | ||
573 | |||
574 | nvgpu_log(g, gpu_dbg_nvlink, | ||
575 | "MINION boot successful: 0x%x", reg); | ||
576 | err = 0; | ||
577 | break; | ||
578 | } | ||
579 | |||
580 | nvgpu_usleep_range(delay, delay * 2); | ||
581 | delay = min_t(unsigned long, | ||
582 | delay << 1, GR_IDLE_CHECK_MAX); | ||
583 | } while (!nvgpu_timeout_expired_msg(&timeout, " minion boot timeout")); | ||
584 | |||
585 | /* Service interrupts */ | ||
586 | gv100_nvlink_minion_falcon_isr(g); | ||
587 | |||
588 | if (nvgpu_timeout_peek_expired(&timeout)) { | ||
589 | err = -ETIMEDOUT; | ||
590 | goto exit; | ||
591 | } | ||
592 | |||
593 | gv100_nvlink_initialize_minion(g); | ||
594 | |||
595 | exit: | ||
596 | if (minion_fw) | ||
597 | nvgpu_release_firmware(g, minion_fw); | ||
598 | |||
599 | return err; | ||
600 | } | ||
601 | |||
602 | /* | ||
603 | * Check if MINION command is complete | ||
604 | */ | ||
605 | static u32 gv100_nvlink_minion_command_complete(struct gk20a *g, u32 link_id) | ||
606 | { | ||
607 | u32 reg; | ||
608 | struct nvgpu_timeout timeout; | ||
609 | u32 delay = GR_IDLE_CHECK_DEFAULT; | ||
610 | |||
611 | |||
612 | nvgpu_timeout_init(g, &timeout, gk20a_get_gr_idle_timeout(g), | ||
613 | NVGPU_TIMER_CPU_TIMER); | ||
614 | |||
615 | do { | ||
616 | reg = MINION_REG_RD32(g, minion_nvlink_dl_cmd_r(link_id)); | ||
617 | |||
618 | if (minion_nvlink_dl_cmd_ready_v(reg) == 1) { | ||
619 | /* Command completed, check sucess */ | ||
620 | if (minion_nvlink_dl_cmd_fault_v(reg) == 1) { | ||
621 | nvgpu_err(g, "minion cmd(%d) error: 0x%x", | ||
622 | link_id, reg); | ||
623 | |||
624 | reg = minion_nvlink_dl_cmd_fault_f(1); | ||
625 | MINION_REG_WR32(g, | ||
626 | minion_nvlink_dl_cmd_r(link_id), reg); | ||
627 | |||
628 | return -EINVAL; | ||
629 | } | ||
630 | |||
631 | /* Commnand success */ | ||
632 | break; | ||
633 | } | ||
634 | nvgpu_usleep_range(delay, delay * 2); | ||
635 | delay = min_t(unsigned long, | ||
636 | delay << 1, GR_IDLE_CHECK_MAX); | ||
637 | |||
638 | } while (!nvgpu_timeout_expired_msg(&timeout, " minion cmd timeout")); | ||
639 | |||
640 | if (nvgpu_timeout_peek_expired(&timeout)) | ||
641 | return -ETIMEDOUT; | ||
642 | |||
643 | nvgpu_log(g, gpu_dbg_nvlink, "minion cmd Complete"); | ||
644 | return 0; | ||
645 | } | ||
646 | |||
647 | /* | ||
648 | * Send Minion command (can be async) | ||
649 | */ | ||
650 | static u32 gv100_nvlink_minion_send_command(struct gk20a *g, | ||
651 | u32 link_id, u32 command, u32 scratch_0, bool sync) | ||
652 | { | ||
653 | u32 err = 0; | ||
654 | |||
655 | /* Check last command succeded */ | ||
656 | err = gv100_nvlink_minion_command_complete(g, link_id); | ||
657 | if (err) | ||
658 | return -EINVAL; | ||
659 | |||
660 | nvgpu_log(g, gpu_dbg_nvlink, | ||
661 | "sending MINION command 0x%x to link %d", command, link_id); | ||
662 | |||
663 | if (command == minion_nvlink_dl_cmd_command_configeom_v()) | ||
664 | MINION_REG_WR32(g, minion_misc_0_r(), | ||
665 | minion_misc_0_scratch_swrw_0_f(scratch_0)); | ||
666 | |||
667 | MINION_REG_WR32(g, minion_nvlink_dl_cmd_r(link_id), | ||
668 | minion_nvlink_dl_cmd_command_f(command) | | ||
669 | minion_nvlink_dl_cmd_fault_f(1)); | ||
670 | |||
671 | if (sync) | ||
672 | err = gv100_nvlink_minion_command_complete(g, link_id); | ||
673 | |||
674 | return err; | ||
675 | } | ||
676 | |||
677 | /* MINION API COMMANDS */ | ||
678 | |||
679 | /* | ||
680 | * Init UPHY | ||
681 | */ | ||
682 | static u32 gv100_nvlink_minion_init_uphy(struct gk20a *g, unsigned long mask, | ||
683 | bool sync) | ||
684 | { | ||
685 | u32 err = 0; | ||
686 | u32 init_pll_cmd; | ||
687 | u32 link_id, master_pll, slave_pll; | ||
688 | u32 master_state, slave_state; | ||
689 | |||
690 | unsigned long link_enable; | ||
691 | |||
692 | switch(g->nvlink.speed) { | ||
693 | case nvgpu_nvlink_speed_default: | ||
694 | init_pll_cmd = minion_nvlink_dl_cmd_command_initpll_0_v(); | ||
695 | break; | ||
696 | default: | ||
697 | nvgpu_err(g, "Unsupported UPHY speed"); | ||
698 | return -EINVAL; | ||
699 | } | ||
700 | |||
701 | link_enable = __gv100_nvlink_get_link_reset_mask(g); | ||
702 | |||
703 | /* Cannot use links 0/1 without refclk buffer */ | ||
704 | if (mask & (BIT(1)|BIT(0))) { | ||
705 | nvgpu_err(g, "links 0/1 not supported on GV100"); | ||
706 | return -EINVAL; | ||
707 | } | ||
708 | |||
709 | for_each_set_bit(link_id, &mask, 32) { | ||
710 | master_pll = g->nvlink.links[link_id].pll_master_link_id; | ||
711 | slave_pll = g->nvlink.links[link_id].pll_slave_link_id; | ||
712 | |||
713 | master_state = nvl_link_state_state_init_v(); | ||
714 | slave_state = nvl_link_state_state_init_v(); | ||
715 | |||
716 | if (BIT(master_pll) & link_enable) | ||
717 | master_state = nvl_link_state_state_v( | ||
718 | g->ops.nvlink.link_get_state(g, master_pll)); | ||
719 | |||
720 | if (BIT(slave_pll) & link_enable) | ||
721 | slave_state = nvl_link_state_state_v( | ||
722 | g->ops.nvlink.link_get_state(g, slave_pll)); | ||
723 | |||
724 | if ((slave_state != nvl_link_state_state_init_v()) || | ||
725 | (master_state != nvl_link_state_state_init_v())) { | ||
726 | nvgpu_err(g, "INIT PLL can only be executed when both " | ||
727 | "master and slave links are in init state"); | ||
728 | return -EINVAL; | ||
729 | } | ||
730 | |||
731 | /* Check if INIT PLL is done on link */ | ||
732 | if (!(BIT(master_pll) & g->nvlink.init_pll_done)) { | ||
733 | err = gv100_nvlink_minion_send_command(g, master_pll, | ||
734 | init_pll_cmd, 0, sync); | ||
735 | if (err) { | ||
736 | nvgpu_err(g, " Error sending INITPLL to minion"); | ||
737 | return err; | ||
738 | } | ||
739 | |||
740 | g->nvlink.init_pll_done |= BIT(master_pll); | ||
741 | } | ||
742 | } | ||
743 | |||
744 | err = gv100_nvlink_setup_pll(g, mask); | ||
745 | if (err) { | ||
746 | nvgpu_err(g, "Error setting up PLL"); | ||
747 | return err; | ||
748 | } | ||
749 | |||
750 | /* INITPHY commands */ | ||
751 | for_each_set_bit(link_id, &mask, 32) { | ||
752 | err = gv100_nvlink_minion_send_command(g, link_id, | ||
753 | minion_nvlink_dl_cmd_command_initphy_v(), 0, sync); | ||
754 | if (err) { | ||
755 | nvgpu_err(g, "Error on INITPHY minion DL command %u", | ||
756 | link_id); | ||
757 | return err; | ||
758 | } | ||
759 | } | ||
760 | |||
761 | return 0; | ||
762 | } | ||
763 | |||
764 | /* | ||
765 | * Configure AC coupling | ||
766 | */ | ||
767 | static u32 gv100_nvlink_minion_configure_ac_coupling(struct gk20a *g, | ||
768 | unsigned long mask, bool sync) | ||
769 | { | ||
770 | u32 err = 0; | ||
771 | u32 i; | ||
772 | u32 temp; | ||
773 | |||
774 | for_each_set_bit(i, &mask, 32) { | ||
775 | |||
776 | temp = DLPL_REG_RD32(g, i, nvl_link_config_r()); | ||
777 | temp &= ~nvl_link_config_ac_safe_en_m(); | ||
778 | temp |= nvl_link_config_ac_safe_en_on_f(); | ||
779 | |||
780 | DLPL_REG_WR32(g, i, nvl_link_config_r(), temp); | ||
781 | |||
782 | err = gv100_nvlink_minion_send_command(g, i, | ||
783 | minion_nvlink_dl_cmd_command_setacmode_v(), 0, sync); | ||
784 | |||
785 | if (err) | ||
786 | return err; | ||
787 | } | ||
788 | |||
789 | return err; | ||
790 | } | ||
791 | |||
792 | /* | ||
793 | * Set Data ready | ||
794 | */ | ||
795 | static u32 gv100_nvlink_minion_data_ready_en(struct gk20a *g, | ||
796 | unsigned long mask, bool sync) | ||
797 | { | ||
798 | u32 err = 0; | ||
799 | u32 i; | ||
800 | |||
801 | for_each_set_bit(i, &mask, 32) { | ||
802 | err = gv100_nvlink_minion_send_command(g, i, | ||
803 | minion_nvlink_dl_cmd_command_initlaneenable_v(), 0, | ||
804 | sync); | ||
805 | if (err) { | ||
806 | nvgpu_err(g, "Failed init lane enable on minion"); | ||
807 | return err; | ||
808 | } | ||
809 | } | ||
810 | |||
811 | for_each_set_bit(i, &mask, 32) { | ||
812 | err = gv100_nvlink_minion_send_command(g, i, | ||
813 | minion_nvlink_dl_cmd_command_initdlpl_v(), 0, sync); | ||
814 | if (err) { | ||
815 | nvgpu_err(g, "Failed init DLPL on minion"); | ||
816 | return err; | ||
817 | } | ||
818 | } | ||
819 | return err; | ||
820 | } | ||
821 | |||
822 | /* | ||
823 | * Request that minion disable the lane | ||
824 | */ | ||
825 | static u32 gv100_nvlink_minion_lane_disable(struct gk20a *g, u32 link_id, | ||
826 | bool sync) | ||
827 | { | ||
828 | u32 err = 0; | ||
829 | |||
830 | err = gv100_nvlink_minion_send_command(g, link_id, | ||
831 | minion_nvlink_dl_cmd_command_lanedisable_v(), 0, sync); | ||
832 | |||
833 | if (err) | ||
834 | nvgpu_err(g, " failed to disable lane on %d", link_id); | ||
835 | |||
836 | return err; | ||
837 | } | ||
838 | |||
839 | /* | ||
840 | * Request that minion shutdown the lane | ||
841 | */ | ||
842 | static u32 gv100_nvlink_minion_lane_shutdown(struct gk20a *g, u32 link_id, | ||
843 | bool sync) | ||
844 | { | ||
845 | u32 err = 0; | ||
846 | |||
847 | err = gv100_nvlink_minion_send_command(g, link_id, | ||
848 | minion_nvlink_dl_cmd_command_laneshutdown_v(), 0, sync); | ||
849 | |||
850 | if (err) | ||
851 | nvgpu_err(g, " failed to shutdown lane on %d", link_id); | ||
852 | |||
853 | return err; | ||
854 | } | ||
855 | |||
856 | |||
857 | |||
858 | /* | ||
859 | *-----------------------------------------------------------------------------* | ||
860 | * TLC API | ||
861 | *-----------------------------------------------------------------------------* | ||
862 | */ | ||
863 | |||
864 | static int gv100_nvlink_get_tlc_reginit(enum nvgpu_nvlink_endp endp, | ||
865 | struct __nvlink_reginit **reg, u32 *count) | ||
866 | { | ||
867 | switch(endp) { | ||
868 | case nvgpu_nvlink_endp_tegra: | ||
869 | *reg = (struct __nvlink_reginit *) | ||
870 | __nvlink_reginit_per_link_tegra; | ||
871 | *count = ARRAY_SIZE(__nvlink_reginit_per_link_tegra); | ||
872 | break; | ||
873 | case nvgpu_nvlink_endp_gpu: | ||
874 | *reg = (struct __nvlink_reginit *) | ||
875 | __nvlink_reginit_per_link_gpu; | ||
876 | *count = ARRAY_SIZE(__nvlink_reginit_per_link_gpu); | ||
877 | break; | ||
878 | default: | ||
879 | return -EINVAL; | ||
880 | } | ||
881 | |||
882 | return 0; | ||
883 | } | ||
884 | |||
885 | /* | ||
886 | * Init TLC IP and prod regs | ||
887 | */ | ||
888 | static void gv100_nvlink_initialize_tlc(struct gk20a *g, u32 link_id) | ||
889 | { | ||
890 | } | ||
891 | |||
892 | /* | ||
893 | * Init TLC per link interrupts | ||
894 | */ | ||
895 | static void gv100_nvlink_tlc_intr_enable(struct gk20a *g, u32 link_id, | ||
896 | bool enable) | ||
897 | { | ||
898 | u32 reg_rx0 = 0, reg_rx1 = 0, reg_tx = 0; | ||
899 | |||
900 | if (enable) { | ||
901 | /* Set PROD values */ | ||
902 | reg_rx0 = 0x0FFFFFF; | ||
903 | reg_rx1 = 0x03FFFFF; | ||
904 | reg_tx = 0x1FFFFFF; | ||
905 | } | ||
906 | |||
907 | TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_0_r(), reg_rx0); | ||
908 | TLC_REG_WR32(g, link_id, nvtlc_rx_err_report_en_1_r(), reg_rx1); | ||
909 | TLC_REG_WR32(g, link_id, nvtlc_tx_err_report_en_0_r(), reg_tx); | ||
910 | } | ||
911 | |||
912 | /* | ||
913 | * helper function to get TLC intr status in common structure | ||
914 | */ | ||
915 | static void gv100_nvlink_tlc_get_intr(struct gk20a *g, u32 link_id) | ||
916 | { | ||
917 | g->nvlink.tlc_rx_err_status_0[link_id] = | ||
918 | TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_0_r()); | ||
919 | g->nvlink.tlc_rx_err_status_1[link_id] = | ||
920 | TLC_REG_RD32(g, link_id, nvtlc_rx_err_status_1_r()); | ||
921 | g->nvlink.tlc_tx_err_status_0[link_id] = | ||
922 | TLC_REG_RD32(g, link_id, nvtlc_tx_err_status_0_r()); | ||
923 | } | ||
924 | |||
925 | /* | ||
926 | * Interrupt routine handler for TLC | ||
927 | */ | ||
928 | static void gv100_nvlink_tlc_isr(struct gk20a *g, u32 link_id) | ||
929 | { | ||
930 | |||
931 | if (g->nvlink.tlc_rx_err_status_0[link_id]) { | ||
932 | /* All TLC RX 0 errors are fatal. Notify and disable */ | ||
933 | nvgpu_err(g, "Fatal TLC RX 0 interrupt on link %d mask: %x", | ||
934 | link_id, g->nvlink.tlc_rx_err_status_0[link_id]); | ||
935 | TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_0_r(), | ||
936 | g->nvlink.tlc_rx_err_status_0[link_id]); | ||
937 | TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_0_r(), | ||
938 | g->nvlink.tlc_rx_err_status_0[link_id]); | ||
939 | } | ||
940 | if (g->nvlink.tlc_rx_err_status_1[link_id]) { | ||
941 | /* All TLC RX 1 errors are fatal. Notify and disable */ | ||
942 | nvgpu_err(g, "Fatal TLC RX 1 interrupt on link %d mask: %x", | ||
943 | link_id, g->nvlink.tlc_rx_err_status_1[link_id]); | ||
944 | TLC_REG_WR32(g, link_id, nvtlc_rx_err_first_1_r(), | ||
945 | g->nvlink.tlc_rx_err_status_1[link_id]); | ||
946 | TLC_REG_WR32(g, link_id, nvtlc_rx_err_status_1_r(), | ||
947 | g->nvlink.tlc_rx_err_status_1[link_id]); | ||
948 | } | ||
949 | if (g->nvlink.tlc_tx_err_status_0[link_id]) { | ||
950 | /* All TLC TX 0 errors are fatal. Notify and disable */ | ||
951 | nvgpu_err(g, "Fatal TLC TX 0 interrupt on link %d mask: %x", | ||
952 | link_id, g->nvlink.tlc_tx_err_status_0[link_id]); | ||
953 | TLC_REG_WR32(g, link_id, nvtlc_tx_err_first_0_r(), | ||
954 | g->nvlink.tlc_tx_err_status_0[link_id]); | ||
955 | TLC_REG_WR32(g, link_id, nvtlc_tx_err_status_0_r(), | ||
956 | g->nvlink.tlc_tx_err_status_0[link_id]); | ||
957 | } | ||
958 | } | ||
959 | |||
960 | /* | ||
961 | *-----------------------------------------------------------------------------* | ||
962 | * DLPL API | ||
963 | *-----------------------------------------------------------------------------* | ||
964 | */ | ||
965 | |||
966 | /* | ||
967 | * DLPL interrupt enable helper | ||
968 | */ | ||
969 | static void gv100_nvlink_dlpl_intr_enable(struct gk20a *g, u32 link_id, | ||
970 | bool enable) | ||
971 | { | ||
972 | u32 reg = 0; | ||
973 | |||
974 | /* Always disable nonstall tree */ | ||
975 | DLPL_REG_WR32(g, link_id, nvl_intr_nonstall_en_r(), 0); | ||
976 | |||
977 | if (!enable) | ||
978 | { | ||
979 | DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), 0); | ||
980 | return; | ||
981 | } | ||
982 | |||
983 | /* Clear interrupt register to get rid of stale state (W1C) */ | ||
984 | DLPL_REG_WR32(g, link_id, nvl_intr_r(), 0xffffffff); | ||
985 | DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff); | ||
986 | |||
987 | reg = nvl_intr_stall_en_tx_recovery_long_enable_f() | | ||
988 | nvl_intr_stall_en_tx_fault_ram_enable_f() | | ||
989 | nvl_intr_stall_en_tx_fault_interface_enable_f() | | ||
990 | nvl_intr_stall_en_tx_fault_sublink_change_enable_f() | | ||
991 | nvl_intr_stall_en_rx_fault_sublink_change_enable_f() | | ||
992 | nvl_intr_stall_en_rx_fault_dl_protocol_enable_f() | | ||
993 | nvl_intr_stall_en_ltssm_fault_enable_f(); | ||
994 | |||
995 | DLPL_REG_WR32(g, link_id, nvl_intr_stall_en_r(), reg); | ||
996 | |||
997 | /* Configure error threshold */ | ||
998 | reg = DLPL_REG_RD32(g, link_id, nvl_sl1_error_rate_ctrl_r()); | ||
999 | reg = set_field(reg, nvl_sl1_error_rate_ctrl_short_threshold_man_m(), | ||
1000 | nvl_sl1_error_rate_ctrl_short_threshold_man_f(0x2)); | ||
1001 | reg = set_field(reg, nvl_sl1_error_rate_ctrl_long_threshold_man_m(), | ||
1002 | nvl_sl1_error_rate_ctrl_long_threshold_man_f(0x2)); | ||
1003 | DLPL_REG_WR32(g, link_id, nvl_sl1_error_rate_ctrl_r(), reg); | ||
1004 | } | ||
1005 | |||
1006 | /* | ||
1007 | * DLPL per-link isr | ||
1008 | */ | ||
1009 | |||
1010 | #define DLPL_NON_FATAL_INTR_MASK (nvl_intr_tx_replay_f(1) | \ | ||
1011 | nvl_intr_tx_recovery_short_f(1) | \ | ||
1012 | nvl_intr_tx_recovery_long_f(1) | \ | ||
1013 | nvl_intr_rx_short_error_rate_f(1) | \ | ||
1014 | nvl_intr_rx_long_error_rate_f(1) | \ | ||
1015 | nvl_intr_rx_ila_trigger_f(1) | \ | ||
1016 | nvl_intr_ltssm_protocol_f(1)) | ||
1017 | |||
1018 | #define DLPL_FATAL_INTR_MASK ( nvl_intr_ltssm_fault_f(1) | \ | ||
1019 | nvl_intr_rx_fault_dl_protocol_f(1) | \ | ||
1020 | nvl_intr_rx_fault_sublink_change_f(1) | \ | ||
1021 | nvl_intr_tx_fault_sublink_change_f(1) | \ | ||
1022 | nvl_intr_tx_fault_interface_f(1) | \ | ||
1023 | nvl_intr_tx_fault_ram_f(1)) | ||
1024 | |||
1025 | static void gv100_nvlink_dlpl_isr(struct gk20a *g, u32 link_id) | ||
1026 | { | ||
1027 | u32 non_fatal_mask = 0; | ||
1028 | u32 fatal_mask = 0; | ||
1029 | u32 intr = 0; | ||
1030 | bool retrain = false; | ||
1031 | u32 err; | ||
1032 | |||
1033 | intr = DLPL_REG_RD32(g, link_id, nvl_intr_r()) & | ||
1034 | DLPL_REG_RD32(g, link_id, nvl_intr_stall_en_r()); | ||
1035 | |||
1036 | if (!intr) | ||
1037 | return; | ||
1038 | |||
1039 | fatal_mask = intr & DLPL_FATAL_INTR_MASK; | ||
1040 | non_fatal_mask = intr & DLPL_NON_FATAL_INTR_MASK; | ||
1041 | |||
1042 | nvgpu_err(g, " handling DLPL %d isr. Fatal: %x non-Fatal: %x", | ||
1043 | link_id, fatal_mask, non_fatal_mask); | ||
1044 | |||
1045 | /* Check if we are not handling an interupt */ | ||
1046 | if ((fatal_mask | non_fatal_mask) & ~intr) | ||
1047 | nvgpu_err(g, "Unable to service DLPL intr on link %d", link_id); | ||
1048 | |||
1049 | if (non_fatal_mask & nvl_intr_tx_recovery_long_f(1)) | ||
1050 | retrain = true; | ||
1051 | if (fatal_mask) | ||
1052 | retrain = false; | ||
1053 | |||
1054 | if (retrain) { | ||
1055 | err = nvgpu_nvlink_train(g, link_id, false); | ||
1056 | if (err) | ||
1057 | nvgpu_err(g, "failed to retrain link %d", link_id); | ||
1058 | } | ||
1059 | |||
1060 | /* Clear interrupts */ | ||
1061 | DLPL_REG_WR32(g, link_id, nvl_intr_r(), (non_fatal_mask | fatal_mask)); | ||
1062 | DLPL_REG_WR32(g, link_id, nvl_intr_sw2_r(), 0xffffffff); | ||
1063 | } | ||
1064 | |||
1065 | /* | ||
1066 | *-----------------------------------------------------------------------------* | ||
1067 | * MIF API | ||
1068 | *-----------------------------------------------------------------------------* | ||
1069 | */ | ||
1070 | |||
1071 | /* | ||
1072 | * Initialize MIF API with PROD settings | ||
1073 | */ | ||
1074 | static void gv100_nvlink_initialize_mif(struct gk20a *g, u32 link_id) | ||
1075 | { | ||
1076 | u32 tmp; | ||
1077 | |||
1078 | /* Enable MIF RX error */ | ||
1079 | |||
1080 | /* Containment (make fatal) */ | ||
1081 | tmp = 0; | ||
1082 | tmp = set_field(tmp, | ||
1083 | ioctrlmif_rx_err_contain_en_0_rxramdataparityerr_m(), | ||
1084 | ioctrlmif_rx_err_contain_en_0_rxramdataparityerr__prod_f()); | ||
1085 | tmp = set_field(tmp, | ||
1086 | ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr_m(), | ||
1087 | ioctrlmif_rx_err_contain_en_0_rxramhdrparityerr__prod_f()); | ||
1088 | MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_contain_en_0_r(), tmp); | ||
1089 | |||
1090 | /* Logging (do not ignore) */ | ||
1091 | tmp = 0; | ||
1092 | tmp = set_field(tmp, | ||
1093 | ioctrlmif_rx_err_log_en_0_rxramdataparityerr_m(), | ||
1094 | ioctrlmif_rx_err_log_en_0_rxramdataparityerr_f(1)); | ||
1095 | tmp = set_field(tmp, | ||
1096 | ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_m(), | ||
1097 | ioctrlmif_rx_err_log_en_0_rxramhdrparityerr_f(1)); | ||
1098 | MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_log_en_0_r(), tmp); | ||
1099 | |||
1100 | /* Tx Error */ | ||
1101 | /* Containment (make fatal) */ | ||
1102 | tmp = 0; | ||
1103 | tmp = set_field(tmp, | ||
1104 | ioctrlmif_tx_err_contain_en_0_txramdataparityerr_m(), | ||
1105 | ioctrlmif_tx_err_contain_en_0_txramdataparityerr__prod_f()); | ||
1106 | tmp = set_field(tmp, | ||
1107 | ioctrlmif_tx_err_contain_en_0_txramhdrparityerr_m(), | ||
1108 | ioctrlmif_tx_err_contain_en_0_txramhdrparityerr__prod_f()); | ||
1109 | MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_contain_en_0_r(), tmp); | ||
1110 | |||
1111 | /* Logging (do not ignore) */ | ||
1112 | tmp = 0; | ||
1113 | tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramdataparityerr_m(), | ||
1114 | ioctrlmif_tx_err_log_en_0_txramdataparityerr_f(1)); | ||
1115 | tmp = set_field(tmp, ioctrlmif_tx_err_log_en_0_txramhdrparityerr_m(), | ||
1116 | ioctrlmif_tx_err_log_en_0_txramhdrparityerr_f(1)); | ||
1117 | MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_log_en_0_r(), tmp); | ||
1118 | |||
1119 | /* Credit release */ | ||
1120 | MIF_REG_WR32(g, link_id, ioctrlmif_rx_ctrl_buffer_ready_r(), 0x1); | ||
1121 | MIF_REG_WR32(g, link_id, ioctrlmif_tx_ctrl_buffer_ready_r(), 0x1); | ||
1122 | } | ||
1123 | |||
1124 | /* | ||
1125 | * Enable per-link MIF interrupts | ||
1126 | */ | ||
1127 | static void gv100_nvlink_mif_intr_enable(struct gk20a *g, u32 link_id, | ||
1128 | bool enable) | ||
1129 | { | ||
1130 | u32 reg0 = 0, reg1 = 0; | ||
1131 | |||
1132 | if (enable) { | ||
1133 | reg0 = set_field(reg0, | ||
1134 | ioctrlmif_rx_err_report_en_0_rxramdataparityerr_m(), | ||
1135 | ioctrlmif_rx_err_report_en_0_rxramdataparityerr_f(1)); | ||
1136 | reg0 = set_field(reg0, | ||
1137 | ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_m(), | ||
1138 | ioctrlmif_rx_err_report_en_0_rxramhdrparityerr_f(1)); | ||
1139 | reg1 = set_field(reg1, | ||
1140 | ioctrlmif_tx_err_report_en_0_txramdataparityerr_m(), | ||
1141 | ioctrlmif_tx_err_report_en_0_txramdataparityerr_f(1)); | ||
1142 | reg1 = set_field(reg1, | ||
1143 | ioctrlmif_tx_err_report_en_0_txramhdrparityerr_m(), | ||
1144 | ioctrlmif_tx_err_report_en_0_txramhdrparityerr_f(1)); | ||
1145 | } | ||
1146 | |||
1147 | MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_report_en_0_r(), reg0); | ||
1148 | MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_report_en_0_r(), reg1); | ||
1149 | } | ||
1150 | |||
1151 | /* | ||
1152 | * Handle per-link MIF interrupts | ||
1153 | */ | ||
1154 | static void gv100_nvlink_mif_isr(struct gk20a *g, u32 link_id) | ||
1155 | { | ||
1156 | u32 intr, fatal_mask = 0; | ||
1157 | |||
1158 | /* RX Errors */ | ||
1159 | intr = MIF_REG_RD32(g, link_id, ioctrlmif_rx_err_status_0_r()); | ||
1160 | if (intr) { | ||
1161 | if (intr & ioctrlmif_rx_err_status_0_rxramdataparityerr_m()) { | ||
1162 | nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_DATA_PARITY", | ||
1163 | link_id); | ||
1164 | fatal_mask |= ioctrlmif_rx_err_status_0_rxramdataparityerr_f(1); | ||
1165 | } | ||
1166 | if (intr & ioctrlmif_rx_err_status_0_rxramhdrparityerr_m()) { | ||
1167 | nvgpu_err(g, "Fatal MIF RX interrupt hit on link %d: RAM_HDR_PARITY", | ||
1168 | link_id); | ||
1169 | fatal_mask |= ioctrlmif_rx_err_status_0_rxramhdrparityerr_f(1); | ||
1170 | } | ||
1171 | |||
1172 | if (fatal_mask) { | ||
1173 | MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_first_0_r(), | ||
1174 | fatal_mask); | ||
1175 | MIF_REG_WR32(g, link_id, ioctrlmif_rx_err_status_0_r(), | ||
1176 | fatal_mask); | ||
1177 | } | ||
1178 | } | ||
1179 | |||
1180 | /* TX Errors */ | ||
1181 | fatal_mask = 0; | ||
1182 | intr = MIF_REG_RD32(g, link_id, ioctrlmif_tx_err_status_0_r()); | ||
1183 | if (intr) { | ||
1184 | if (intr & ioctrlmif_tx_err_status_0_txramdataparityerr_m()) { | ||
1185 | nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_DATA_PARITY", | ||
1186 | link_id); | ||
1187 | fatal_mask |= ioctrlmif_tx_err_status_0_txramdataparityerr_f(1); | ||
1188 | } | ||
1189 | if (intr & ioctrlmif_tx_err_status_0_txramhdrparityerr_m()) { | ||
1190 | nvgpu_err(g, "Fatal MIF TX interrupt hit on link %d: RAM_HDR_PARITY", | ||
1191 | link_id); | ||
1192 | fatal_mask |= ioctrlmif_tx_err_status_0_txramhdrparityerr_f(1); | ||
1193 | } | ||
1194 | |||
1195 | if (fatal_mask) { | ||
1196 | MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_first_0_r(), | ||
1197 | fatal_mask); | ||
1198 | MIF_REG_WR32(g, link_id, ioctrlmif_tx_err_status_0_r(), | ||
1199 | fatal_mask); | ||
1200 | } | ||
1201 | } | ||
1202 | } | ||
1203 | |||
1204 | /* | ||
1205 | *-----------------------------------------------------------------------------* | ||
1206 | * NVLIPT API | ||
1207 | *-----------------------------------------------------------------------------* | ||
1208 | */ | ||
1209 | |||
1210 | /* | ||
1211 | * NVLIPT IP initialization (per-link) | ||
1212 | */ | ||
1213 | static void gv100_nvlink_initialize_nvlipt(struct gk20a *g, u32 link_id) | ||
1214 | { | ||
1215 | /* init persistent scratch registers */ | ||
1216 | IPT_REG_WR32(g, nvlipt_scratch_cold_r(), | ||
1217 | nvlipt_scratch_cold_data_init_v()); | ||
1218 | |||
1219 | /* | ||
1220 | * AErr settings (top level) | ||
1221 | */ | ||
1222 | |||
1223 | /* UC first and status reg (W1C) need to be cleared byt arch */ | ||
1224 | IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); | ||
1225 | IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); | ||
1226 | |||
1227 | /* AErr Severity */ | ||
1228 | IPT_REG_WR32(g, IPT_ERR_UC_SEVERITY_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); | ||
1229 | |||
1230 | /* AErr Control settings */ | ||
1231 | IPT_REG_WR32(g, IPT_ERR_CONTROL_LINK(link_id), | ||
1232 | nvlipt_err_control_link0_fatalenable_f(1) | | ||
1233 | nvlipt_err_control_link0_nonfatalenable_f(1)); | ||
1234 | } | ||
1235 | |||
1236 | /* | ||
1237 | * Enable NVLIPT interrupts | ||
1238 | */ | ||
1239 | static void gv100_nvlink_nvlipt_intr_enable(struct gk20a *g, u32 link_id, | ||
1240 | bool enable) | ||
1241 | { | ||
1242 | u32 val = 0; | ||
1243 | u32 reg; | ||
1244 | |||
1245 | if (enable) | ||
1246 | val = 1; | ||
1247 | |||
1248 | reg = IPT_REG_RD32(g, IPT_INTR_CONTROL_LINK(link_id)); | ||
1249 | reg = set_field(reg, nvlipt_intr_control_link0_stallenable_m(), | ||
1250 | nvlipt_intr_control_link0_stallenable_f(val)); | ||
1251 | reg = set_field(reg, nvlipt_intr_control_link0_nostallenable_m(), | ||
1252 | nvlipt_intr_control_link0_nostallenable_f(val)); | ||
1253 | IPT_REG_WR32(g, IPT_INTR_CONTROL_LINK(link_id), reg); | ||
1254 | } | ||
1255 | |||
1256 | /* | ||
1257 | * Per-link NVLIPT ISR handler | ||
1258 | */ | ||
1259 | bool gv100_nvlink_nvlipt_isr(struct gk20a *g, u32 link_id) | ||
1260 | { | ||
1261 | /* | ||
1262 | * Interrupt handling happens in leaf handlers. Assume all interrupts | ||
1263 | * were handled and clear roll ups/ | ||
1264 | */ | ||
1265 | IPT_REG_WR32(g, IPT_ERR_UC_FIRST_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); | ||
1266 | IPT_REG_WR32(g, IPT_ERR_UC_STATUS_LINK(link_id), IPT_ERR_UC_ACTIVE_BITS); | ||
1267 | |||
1268 | return true; | ||
1269 | } | ||
1270 | |||
1271 | /* | ||
1272 | ******************************************************************************* | ||
1273 | * Interrupt handling functions * | ||
1274 | ******************************************************************************* | ||
1275 | */ | ||
1276 | |||
1277 | /* | ||
1278 | * Enable common interrupts | ||
1279 | */ | ||
1280 | static void gv100_nvlink_common_intr_enable(struct gk20a *g, | ||
1281 | unsigned long mask) | ||
1282 | { | ||
1283 | u32 reg, i; | ||
1284 | |||
1285 | /* Init HS HUB SW state */ | ||
1286 | g->nvlink.hshub_config0 = gk20a_readl(g, fb_hshub_config0_r()); | ||
1287 | g->nvlink.hshub_config1 = gk20a_readl(g, fb_hshub_config1_r()); | ||
1288 | g->nvlink.hshub_config2 = gk20a_readl(g, fb_hshub_config2_r()); | ||
1289 | g->nvlink.hshub_config6 = gk20a_readl(g, fb_hshub_config6_r()); | ||
1290 | |||
1291 | /* Init IOCTRL */ | ||
1292 | for_each_set_bit(i, &mask, 32) { | ||
1293 | reg = IOCTRL_REG_RD32(g, ioctrl_link_intr_0_mask_r(i)); | ||
1294 | reg |= (ioctrl_link_intr_0_mask_fatal_f(1) | | ||
1295 | ioctrl_link_intr_0_mask_nonfatal_f(1) | | ||
1296 | ioctrl_link_intr_0_mask_correctable_f(1) | | ||
1297 | ioctrl_link_intr_0_mask_intra_f(1)); | ||
1298 | IOCTRL_REG_WR32(g, ioctrl_link_intr_0_mask_r(i), reg); | ||
1299 | } | ||
1300 | |||
1301 | reg = IOCTRL_REG_RD32(g, ioctrl_common_intr_0_mask_r()); | ||
1302 | reg |= (ioctrl_common_intr_0_mask_fatal_f(1) | | ||
1303 | ioctrl_common_intr_0_mask_nonfatal_f(1) | | ||
1304 | ioctrl_common_intr_0_mask_correctable_f(1) | | ||
1305 | ioctrl_common_intr_0_mask_intra_f(1)); | ||
1306 | IOCTRL_REG_WR32(g, ioctrl_common_intr_0_mask_r(), reg); | ||
1307 | |||
1308 | /* Init NVLIPT */ | ||
1309 | IPT_REG_WR32(g, nvlipt_intr_control_common_r(), | ||
1310 | nvlipt_intr_control_common_stallenable_f(1) | | ||
1311 | nvlipt_intr_control_common_nonstallenable_f(1)); | ||
1312 | } | ||
1313 | |||
1314 | /* | ||
1315 | * Enable link specific interrupts (top-level) | ||
1316 | */ | ||
1317 | static void gv100_nvlink_enable_link_intr(struct gk20a *g, u32 link_id, | ||
1318 | bool enable) | ||
1319 | { | ||
1320 | gv100_nvlink_minion_link_intr_enable(g, link_id, enable); | ||
1321 | gv100_nvlink_dlpl_intr_enable(g, link_id, enable); | ||
1322 | gv100_nvlink_tlc_intr_enable(g, link_id, enable); | ||
1323 | gv100_nvlink_mif_intr_enable(g, link_id, enable); | ||
1324 | gv100_nvlink_nvlipt_intr_enable(g, link_id, enable); | ||
1325 | } | ||
1326 | |||
1327 | /* | ||
1328 | * Top level interrupt handler | ||
1329 | */ | ||
1330 | int gv100_nvlink_isr(struct gk20a *g) | ||
1331 | { | ||
1332 | unsigned long links; | ||
1333 | u32 link_id; | ||
1334 | |||
1335 | links = ioctrl_top_intr_0_status_link_v( | ||
1336 | IOCTRL_REG_RD32(g, ioctrl_top_intr_0_status_r())); | ||
1337 | |||
1338 | links &= g->nvlink.enabled_links; | ||
1339 | /* As per ARCH minion must be serviced first */ | ||
1340 | gv100_nvlink_minion_isr(g); | ||
1341 | |||
1342 | for_each_set_bit(link_id, &links, 32) { | ||
1343 | /* Cache error logs from TLC, DL handler may clear them */ | ||
1344 | gv100_nvlink_tlc_get_intr(g, link_id); | ||
1345 | gv100_nvlink_dlpl_isr(g, link_id); | ||
1346 | gv100_nvlink_tlc_isr(g, link_id); | ||
1347 | gv100_nvlink_mif_isr(g, link_id); | ||
1348 | |||
1349 | /* NVLIPT is top-level. Do it last */ | ||
1350 | gv100_nvlink_nvlipt_isr(g, link_id); | ||
1351 | } | ||
1352 | return 0; | ||
1353 | } | ||
1354 | |||
1355 | /******************************************************************************* | ||
1356 | * Helper functions * | ||
1357 | ******************************************************************************* | ||
1358 | */ | ||
1359 | |||
1360 | static u32 __gv100_nvlink_get_link_reset_mask(struct gk20a *g) | ||
1361 | { | ||
1362 | u32 reg_data; | ||
1363 | |||
1364 | reg_data = IOCTRL_REG_RD32(g, ioctrl_reset_r()); | ||
1365 | |||
1366 | return ioctrl_reset_linkreset_v(reg_data); | ||
1367 | } | ||
1368 | |||
1369 | static u32 __gv100_nvlink_state_load_hal(struct gk20a *g) | ||
1370 | { | ||
1371 | unsigned long discovered = g->nvlink.discovered_links; | ||
1372 | |||
1373 | gv100_nvlink_common_intr_enable(g, discovered); | ||
1374 | return gv100_nvlink_minion_load(g); | ||
1375 | } | ||
1376 | |||
1377 | #define TRIM_SYS_NVLINK_CTRL(i) (trim_sys_nvlink0_ctrl_r() + 16*i) | ||
1378 | #define TRIM_SYS_NVLINK_STATUS(i) (trim_sys_nvlink0_status_r() + 16*i) | ||
1379 | |||
1380 | static int gv100_nvlink_setup_pll(struct gk20a *g, unsigned long mask) | ||
1381 | { | ||
1382 | u32 reg; | ||
1383 | u32 i; | ||
1384 | u32 links_off; | ||
1385 | struct nvgpu_timeout timeout; | ||
1386 | u32 pad_ctrl, swap_ctrl; | ||
1387 | u32 pll_id; | ||
1388 | |||
1389 | reg = gk20a_readl(g, trim_sys_nvlink_uphy_cfg_r()); | ||
1390 | reg = set_field(reg, trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_m(), | ||
1391 | trim_sys_nvlink_uphy_cfg_phy2clks_use_lockdet_f(1)); | ||
1392 | gk20a_writel(g, trim_sys_nvlink_uphy_cfg_r(), reg); | ||
1393 | |||
1394 | reg = gk20a_readl(g, top_nvhsclk_ctrl_r()); | ||
1395 | |||
1396 | pad_ctrl = top_nvhsclk_ctrl_e_clk_nvl_v(reg); | ||
1397 | swap_ctrl = top_nvhsclk_ctrl_swap_clk_nvl_v(reg); | ||
1398 | |||
1399 | for_each_set_bit(i, &mask, 32) { | ||
1400 | /* There are 3 PLLs for 6 links. We have 3 bits for each PLL. | ||
1401 | * The PLL bit corresponding to a link is /2 of its master link. | ||
1402 | */ | ||
1403 | pll_id = g->nvlink.links[i].pll_master_link_id >> 1; | ||
1404 | pad_ctrl |= BIT(pll_id); | ||
1405 | swap_ctrl |= BIT(pll_id); | ||
1406 | } | ||
1407 | |||
1408 | reg = set_field(reg, top_nvhsclk_ctrl_e_clk_nvl_m(), | ||
1409 | top_nvhsclk_ctrl_e_clk_nvl_f(pad_ctrl)); | ||
1410 | reg = set_field(reg, top_nvhsclk_ctrl_e_clk_core_m(), | ||
1411 | top_nvhsclk_ctrl_e_clk_core_f(0x1)); | ||
1412 | reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_nvl_m(), | ||
1413 | top_nvhsclk_ctrl_swap_clk_nvl_f(swap_ctrl)); | ||
1414 | reg = set_field(reg, top_nvhsclk_ctrl_swap_clk_core_m(), | ||
1415 | top_nvhsclk_ctrl_swap_clk_core_f(0x0)); | ||
1416 | |||
1417 | gk20a_writel(g, top_nvhsclk_ctrl_r(), reg); | ||
1418 | |||
1419 | for_each_set_bit(i, &mask, 32) { | ||
1420 | reg = gk20a_readl(g, TRIM_SYS_NVLINK_CTRL(i)); | ||
1421 | reg = set_field(reg, | ||
1422 | trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_m(), | ||
1423 | trim_sys_nvlink0_ctrl_unit2clks_pll_turn_off_f(0)); | ||
1424 | gk20a_writel(g, TRIM_SYS_NVLINK_CTRL(i), reg); | ||
1425 | } | ||
1426 | |||
1427 | /* Poll for links to go up */ | ||
1428 | links_off = mask; | ||
1429 | |||
1430 | nvgpu_timeout_init(g, &timeout, | ||
1431 | NVLINK_PLL_ON_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); | ||
1432 | do { | ||
1433 | for_each_set_bit(i, &mask, 32) { | ||
1434 | reg = gk20a_readl(g, TRIM_SYS_NVLINK_STATUS(i)); | ||
1435 | if (trim_sys_nvlink0_status_pll_off_v(reg) == 0) | ||
1436 | links_off &= ~BIT(i); | ||
1437 | } | ||
1438 | nvgpu_udelay(5); | ||
1439 | |||
1440 | } while((!nvgpu_timeout_expired_msg(&timeout, "timeout on pll on")) && | ||
1441 | links_off); | ||
1442 | |||
1443 | if (nvgpu_timeout_peek_expired(&timeout)) | ||
1444 | return -ETIMEDOUT; | ||
1445 | |||
1446 | return 0; | ||
1447 | } | ||
1448 | |||
1449 | static void gv100_nvlink_prog_alt_clk(struct gk20a *g) | ||
1450 | { | ||
1451 | u32 tmp; | ||
1452 | |||
1453 | /* RMW registers need to be separate */ | ||
1454 | tmp = gk20a_readl(g, trim_sys_nvl_common_clk_alt_switch_r()); | ||
1455 | tmp &= ~trim_sys_nvl_common_clk_alt_switch_slowclk_m(); | ||
1456 | tmp |= trim_sys_nvl_common_clk_alt_switch_slowclk_xtal4x_f(); | ||
1457 | gk20a_writel(g, trim_sys_nvl_common_clk_alt_switch_r(), tmp); | ||
1458 | |||
1459 | tmp &= ~trim_sys_nvl_common_clk_alt_switch_finalsel_m(); | ||
1460 | tmp |= trim_sys_nvl_common_clk_alt_switch_finalsel_slowclk_f(); | ||
1461 | } | ||
1462 | |||
1463 | static int gv100_nvlink_enable_links_pre_top(struct gk20a *g, u32 links) | ||
1464 | { | ||
1465 | u32 link_id; | ||
1466 | unsigned long enabled_links = links; | ||
1467 | u32 tmp; | ||
1468 | u32 reg; | ||
1469 | u32 delay = ioctrl_reset_sw_post_reset_delay_microseconds_v(); | ||
1470 | u32 err; | ||
1471 | |||
1472 | nvgpu_log(g, gpu_dbg_nvlink, " enabling 0x%lx links", enabled_links); | ||
1473 | /* Take links out of reset */ | ||
1474 | for_each_set_bit(link_id, &enabled_links, 32) { | ||
1475 | reg = IOCTRL_REG_RD32(g, ioctrl_reset_r()); | ||
1476 | |||
1477 | tmp = (BIT(link_id) | | ||
1478 | BIT(g->nvlink.links[link_id].pll_master_link_id)); | ||
1479 | |||
1480 | reg = set_field(reg, ioctrl_reset_linkreset_m(), | ||
1481 | ioctrl_reset_linkreset_f( ioctrl_reset_linkreset_v(reg) | | ||
1482 | tmp)); | ||
1483 | |||
1484 | IOCTRL_REG_WR32(g, ioctrl_reset_r(), reg); | ||
1485 | nvgpu_udelay(delay); | ||
1486 | |||
1487 | reg = IOCTRL_REG_RD32(g, ioctrl_debug_reset_r()); | ||
1488 | |||
1489 | reg &= ~ioctrl_debug_reset_link_f(BIT(link_id)); | ||
1490 | IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); | ||
1491 | nvgpu_udelay(delay); | ||
1492 | |||
1493 | reg |= ioctrl_debug_reset_link_f(BIT(link_id)); | ||
1494 | IOCTRL_REG_WR32(g, ioctrl_debug_reset_r(), reg); | ||
1495 | nvgpu_udelay(delay); | ||
1496 | |||
1497 | /* Enable Link DLPL for AN0 */ | ||
1498 | reg = DLPL_REG_RD32(g, link_id, nvl_link_config_r()); | ||
1499 | reg = set_field(reg, nvl_link_config_link_en_m(), | ||
1500 | nvl_link_config_link_en_f(1)); | ||
1501 | DLPL_REG_WR32(g, link_id, nvl_link_config_r(), reg); | ||
1502 | |||
1503 | /* This should be done by the NVLINK API */ | ||
1504 | err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true); | ||
1505 | if (err) { | ||
1506 | nvgpu_err(g, "Failed to init phy of link: %u", link_id); | ||
1507 | return err; | ||
1508 | } | ||
1509 | |||
1510 | err = gv100_nvlink_rxcal_en(g, BIT(link_id)); | ||
1511 | if (err) { | ||
1512 | nvgpu_err(g, "Failed to RXcal on link: %u", link_id); | ||
1513 | return err; | ||
1514 | } | ||
1515 | |||
1516 | err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true); | ||
1517 | if (err) { | ||
1518 | nvgpu_err(g, "Failed to set data ready link:%u", | ||
1519 | link_id); | ||
1520 | return err; | ||
1521 | } | ||
1522 | |||
1523 | g->nvlink.enabled_links |= BIT(link_id); | ||
1524 | } | ||
1525 | |||
1526 | nvgpu_log(g, gpu_dbg_nvlink, "enabled_links=0x%08x", | ||
1527 | g->nvlink.enabled_links); | ||
1528 | |||
1529 | if (g->nvlink.enabled_links) | ||
1530 | return 0; | ||
1531 | |||
1532 | nvgpu_err(g, " No links were enabled"); | ||
1533 | return -EINVAL; | ||
1534 | } | ||
1535 | |||
1536 | static int gv100_nvlink_enable_links_post_top(struct gk20a *g, u32 links) | ||
1537 | { | ||
1538 | u32 link_id; | ||
1539 | unsigned long enabled_links = (links & g->nvlink.enabled_links) & | ||
1540 | ~g->nvlink.initialized_links; | ||
1541 | u32 reg; | ||
1542 | |||
1543 | for_each_set_bit(link_id, &enabled_links, 32) { | ||
1544 | |||
1545 | /* WAR for HW bug 1888034 */ | ||
1546 | reg = DLPL_REG_RD32(g, link_id, nvl_sl0_safe_ctrl2_tx_r()); | ||
1547 | reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_init_m(), | ||
1548 | nvl_sl0_safe_ctrl2_tx_ctr_init_init_f()); | ||
1549 | reg = set_field(reg, nvl_sl0_safe_ctrl2_tx_ctr_initscl_m(), | ||
1550 | nvl_sl0_safe_ctrl2_tx_ctr_initscl_init_f()); | ||
1551 | DLPL_REG_WR32(g, link_id, nvl_sl0_safe_ctrl2_tx_r(), reg); | ||
1552 | |||
1553 | gv100_nvlink_initialize_tlc(g, link_id); | ||
1554 | gv100_nvlink_initialize_nvlipt(g, link_id); | ||
1555 | gv100_nvlink_enable_link_intr(g, link_id, true); | ||
1556 | |||
1557 | g->nvlink.initialized_links |= BIT(link_id); | ||
1558 | }; | ||
1559 | |||
1560 | return 0; | ||
1561 | } | ||
1562 | |||
1563 | static u32 gv100_nvlink_prbs_gen_en(struct gk20a *g, unsigned long mask) | ||
1564 | { | ||
1565 | u32 reg; | ||
1566 | u32 link_id; | ||
1567 | |||
1568 | for_each_set_bit(link_id, &mask, 32) { | ||
1569 | /* Write is required as part of HW sequence */ | ||
1570 | DLPL_REG_WR32(g, link_id, nvl_sl1_rxslsm_timeout_2_r(), 0); | ||
1571 | |||
1572 | reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r()); | ||
1573 | reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(), | ||
1574 | nvl_txiobist_config_dpg_prbsseedld_f(0x1)); | ||
1575 | DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg); | ||
1576 | |||
1577 | reg = DLPL_REG_RD32(g, link_id, nvl_txiobist_config_r()); | ||
1578 | reg = set_field(reg, nvl_txiobist_config_dpg_prbsseedld_m(), | ||
1579 | nvl_txiobist_config_dpg_prbsseedld_f(0x0)); | ||
1580 | DLPL_REG_WR32(g, link_id, nvl_txiobist_config_r(), reg); | ||
1581 | } | ||
1582 | |||
1583 | return 0; | ||
1584 | } | ||
1585 | |||
1586 | static u32 gv100_nvlink_rxcal_en(struct gk20a *g, unsigned long mask) | ||
1587 | { | ||
1588 | u32 link_id; | ||
1589 | struct nvgpu_timeout timeout; | ||
1590 | u32 reg; | ||
1591 | |||
1592 | for_each_set_bit(link_id, &mask, 32) { | ||
1593 | /* Timeout from HW specs */ | ||
1594 | nvgpu_timeout_init(g, &timeout, | ||
1595 | 8*NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); | ||
1596 | reg = DLPL_REG_RD32(g, link_id, nvl_br0_cfg_cal_r()); | ||
1597 | reg = set_field(reg, nvl_br0_cfg_cal_rxcal_m(), | ||
1598 | nvl_br0_cfg_cal_rxcal_on_f()); | ||
1599 | DLPL_REG_WR32(g, link_id, nvl_br0_cfg_cal_r(), reg); | ||
1600 | |||
1601 | do { | ||
1602 | reg = DLPL_REG_RD32(g, link_id, | ||
1603 | nvl_br0_cfg_status_cal_r()); | ||
1604 | |||
1605 | if (nvl_br0_cfg_status_cal_rxcal_done_v(reg) == 1) | ||
1606 | break; | ||
1607 | nvgpu_udelay(5); | ||
1608 | } while(!nvgpu_timeout_expired_msg(&timeout, | ||
1609 | "timeout on rxcal")); | ||
1610 | |||
1611 | if (nvgpu_timeout_peek_expired(&timeout)) | ||
1612 | return -ETIMEDOUT; | ||
1613 | } | ||
1614 | |||
1615 | return 0; | ||
1616 | } | ||
1617 | |||
1618 | /* | ||
1619 | ******************************************************************************* | ||
1620 | * Internal "ops" functions * | ||
1621 | ******************************************************************************* | ||
1622 | */ | ||
1623 | |||
1624 | |||
1625 | /* | ||
1626 | * Main Nvlink init function. Calls into the Nvlink core API | ||
1627 | */ | ||
1628 | int gv100_nvlink_init(struct gk20a *g) | ||
1629 | { | ||
1630 | int err = 0; | ||
1631 | |||
1632 | if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) | ||
1633 | return -ENODEV; | ||
1634 | |||
1635 | err = nvgpu_nvlink_enumerate(g); | ||
1636 | if (err) | ||
1637 | return err; | ||
1638 | |||
1639 | /* Set HSHUB and SG_PHY */ | ||
1640 | __nvgpu_set_enabled(g, NVGPU_MM_USE_PHYSICAL_SG, true); | ||
1641 | return err; | ||
1642 | } | ||
1643 | |||
1644 | /* | ||
1645 | * Query internal device topology and discover devices in nvlink local | ||
1646 | * infrastructure. Initialize register base and offsets | ||
1647 | */ | ||
1648 | int gv100_nvlink_discover_link(struct gk20a *g) | ||
1649 | { | ||
1650 | u32 i; | ||
1651 | u32 ioctrl_entry_addr; | ||
1652 | u8 ioctrl_device_type; | ||
1653 | u32 table_entry; | ||
1654 | u32 ioctrl_info_entry_type; | ||
1655 | u8 ioctrl_discovery_size; | ||
1656 | bool is_chain = false; | ||
1657 | u8 nvlink_num_devices = 0; | ||
1658 | unsigned long available_links = 0; | ||
1659 | struct nvgpu_nvlink_device_list *device_table; | ||
1660 | u32 err = 0; | ||
1661 | |||
1662 | /* | ||
1663 | * Process Entry 0 & 1 of IOCTRL table to find table size | ||
1664 | */ | ||
1665 | if (g->nvlink.ioctrl_table && g->nvlink.ioctrl_table[0].pri_base_addr) { | ||
1666 | ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr; | ||
1667 | table_entry = gk20a_readl(g, ioctrl_entry_addr); | ||
1668 | ioctrl_info_entry_type = nvlinkip_discovery_common_device_v(table_entry); | ||
1669 | } else { | ||
1670 | nvgpu_err(g, " Bad IOCTRL PRI Base addr"); | ||
1671 | return -EINVAL; | ||
1672 | } | ||
1673 | |||
1674 | if (ioctrl_info_entry_type == NVL_DEVICE(ioctrl)) { | ||
1675 | ioctrl_entry_addr = g->nvlink.ioctrl_table[0].pri_base_addr + 4; | ||
1676 | table_entry = gk20a_readl(g, ioctrl_entry_addr); | ||
1677 | ioctrl_discovery_size = nvlinkip_discovery_common_ioctrl_length_v(table_entry); | ||
1678 | nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL size: %d", ioctrl_discovery_size); | ||
1679 | } else { | ||
1680 | nvgpu_err(g, " First entry of IOCTRL_DISCOVERY invalid"); | ||
1681 | return -EINVAL; | ||
1682 | } | ||
1683 | |||
1684 | device_table = nvgpu_kzalloc(g, ioctrl_discovery_size * | ||
1685 | sizeof(struct nvgpu_nvlink_device_list)); | ||
1686 | if (!device_table) { | ||
1687 | nvgpu_err(g, " Unable to allocate nvlink device table"); | ||
1688 | return -ENOMEM; | ||
1689 | } | ||
1690 | |||
1691 | for (i = 0; i < ioctrl_discovery_size; i++) { | ||
1692 | ioctrl_entry_addr = | ||
1693 | g->nvlink.ioctrl_table[0].pri_base_addr + 4*i; | ||
1694 | table_entry = gk20a_readl(g, ioctrl_entry_addr); | ||
1695 | |||
1696 | nvgpu_log(g, gpu_dbg_nvlink, "parsing ioctrl %d: 0x%08x", i, table_entry); | ||
1697 | |||
1698 | ioctrl_info_entry_type = nvlinkip_discovery_common_entry_v(table_entry); | ||
1699 | |||
1700 | if (ioctrl_info_entry_type == | ||
1701 | nvlinkip_discovery_common_entry_invalid_v()) | ||
1702 | continue; | ||
1703 | |||
1704 | if (ioctrl_info_entry_type == | ||
1705 | nvlinkip_discovery_common_entry_enum_v()) { | ||
1706 | |||
1707 | nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is ENUM", i); | ||
1708 | |||
1709 | ioctrl_device_type = | ||
1710 | nvlinkip_discovery_common_device_v(table_entry); | ||
1711 | |||
1712 | if (nvlinkip_discovery_common_chain_v(table_entry) != | ||
1713 | nvlinkip_discovery_common_chain_enable_v()) { | ||
1714 | |||
1715 | nvgpu_log(g, gpu_dbg_nvlink, | ||
1716 | "IOCTRL entry %d is ENUM but no chain", | ||
1717 | i); | ||
1718 | err = -EINVAL; | ||
1719 | break; | ||
1720 | } | ||
1721 | |||
1722 | is_chain = true; | ||
1723 | device_table[nvlink_num_devices].valid = true; | ||
1724 | device_table[nvlink_num_devices].device_type = | ||
1725 | ioctrl_device_type; | ||
1726 | device_table[nvlink_num_devices].device_id = | ||
1727 | nvlinkip_discovery_common_id_v(table_entry); | ||
1728 | device_table[nvlink_num_devices].device_version = | ||
1729 | nvlinkip_discovery_common_version_v( | ||
1730 | table_entry); | ||
1731 | continue; | ||
1732 | } | ||
1733 | |||
1734 | if (ioctrl_info_entry_type == | ||
1735 | nvlinkip_discovery_common_entry_data1_v()) { | ||
1736 | nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA1", i); | ||
1737 | |||
1738 | if (is_chain) { | ||
1739 | device_table[nvlink_num_devices].pri_base_addr = | ||
1740 | nvlinkip_discovery_common_pri_base_v( | ||
1741 | table_entry) << 12; | ||
1742 | |||
1743 | device_table[nvlink_num_devices].intr_enum = | ||
1744 | nvlinkip_discovery_common_intr_v( | ||
1745 | table_entry); | ||
1746 | |||
1747 | device_table[nvlink_num_devices].reset_enum = | ||
1748 | nvlinkip_discovery_common_reset_v( | ||
1749 | table_entry); | ||
1750 | |||
1751 | nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d type = %d base: 0x%08x intr: %d reset: %d", | ||
1752 | i, | ||
1753 | device_table[nvlink_num_devices].device_type, | ||
1754 | device_table[nvlink_num_devices].pri_base_addr, | ||
1755 | device_table[nvlink_num_devices].intr_enum, | ||
1756 | device_table[nvlink_num_devices].reset_enum); | ||
1757 | |||
1758 | if (device_table[nvlink_num_devices].device_type == | ||
1759 | NVL_DEVICE(dlpl)) { | ||
1760 | device_table[nvlink_num_devices].num_tx = | ||
1761 | nvlinkip_discovery_common_dlpl_num_tx_v(table_entry); | ||
1762 | device_table[nvlink_num_devices].num_rx = | ||
1763 | nvlinkip_discovery_common_dlpl_num_rx_v(table_entry); | ||
1764 | |||
1765 | nvgpu_log(g, gpu_dbg_nvlink, "DLPL tx: %d rx: %d", | ||
1766 | device_table[nvlink_num_devices].num_tx, | ||
1767 | device_table[nvlink_num_devices].num_rx); | ||
1768 | } | ||
1769 | |||
1770 | if (nvlinkip_discovery_common_chain_v(table_entry) != | ||
1771 | nvlinkip_discovery_common_chain_enable_v()) { | ||
1772 | |||
1773 | is_chain = false; | ||
1774 | nvlink_num_devices++; | ||
1775 | } | ||
1776 | } | ||
1777 | continue; | ||
1778 | } | ||
1779 | |||
1780 | if (ioctrl_info_entry_type == | ||
1781 | nvlinkip_discovery_common_entry_data2_v()) { | ||
1782 | |||
1783 | nvgpu_log(g, gpu_dbg_nvlink, "IOCTRL entry %d is DATA2", i); | ||
1784 | |||
1785 | if (is_chain) { | ||
1786 | if (nvlinkip_discovery_common_dlpl_data2_type_v(table_entry)) { | ||
1787 | device_table[nvlink_num_devices].pll_master = | ||
1788 | nvlinkip_discovery_common_dlpl_data2_master_v(table_entry); | ||
1789 | device_table[nvlink_num_devices].pll_master_id = | ||
1790 | nvlinkip_discovery_common_dlpl_data2_masterid_v(table_entry); | ||
1791 | nvgpu_log(g, gpu_dbg_nvlink, "PLL info: Master: %d, Master ID: %d", | ||
1792 | device_table[nvlink_num_devices].pll_master, | ||
1793 | device_table[nvlink_num_devices].pll_master_id); | ||
1794 | } | ||
1795 | |||
1796 | if (nvlinkip_discovery_common_chain_v(table_entry) != | ||
1797 | nvlinkip_discovery_common_chain_enable_v()) { | ||
1798 | |||
1799 | is_chain = false; | ||
1800 | nvlink_num_devices++; | ||
1801 | } | ||
1802 | } | ||
1803 | continue; | ||
1804 | } | ||
1805 | } | ||
1806 | |||
1807 | g->nvlink.device_table = device_table; | ||
1808 | g->nvlink.num_devices = nvlink_num_devices; | ||
1809 | |||
1810 | /* | ||
1811 | * Print table | ||
1812 | */ | ||
1813 | for (i = 0; i < nvlink_num_devices; i++) { | ||
1814 | if (device_table[i].valid) { | ||
1815 | nvgpu_log(g, gpu_dbg_nvlink, "Device %d - %s", i, | ||
1816 | __gv100_device_type_to_str( | ||
1817 | device_table[i].device_type)); | ||
1818 | nvgpu_log(g, gpu_dbg_nvlink, "+Link/Device Id: %d", device_table[i].device_id); | ||
1819 | nvgpu_log(g, gpu_dbg_nvlink, "+Version: %d", device_table[i].device_version); | ||
1820 | nvgpu_log(g, gpu_dbg_nvlink, "+Base Addr: 0x%08x", device_table[i].pri_base_addr); | ||
1821 | nvgpu_log(g, gpu_dbg_nvlink, "+Intr Enum: %d", device_table[i].intr_enum); | ||
1822 | nvgpu_log(g, gpu_dbg_nvlink, "+Reset Enum: %d", device_table[i].reset_enum); | ||
1823 | if ((device_table[i].device_type == NVL_DEVICE(dlpl)) || | ||
1824 | (device_table[i].device_type == NVL_DEVICE(nvlink))) { | ||
1825 | nvgpu_log(g, gpu_dbg_nvlink, "+TX: %d", device_table[i].num_tx); | ||
1826 | nvgpu_log(g, gpu_dbg_nvlink, "+RX: %d", device_table[i].num_rx); | ||
1827 | nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master: %d", device_table[i].pll_master); | ||
1828 | nvgpu_log(g, gpu_dbg_nvlink, "+PLL Master ID: %d", device_table[i].pll_master_id); | ||
1829 | } | ||
1830 | } | ||
1831 | } | ||
1832 | |||
1833 | for (i = 0; i < nvlink_num_devices; i++) { | ||
1834 | if (device_table[i].valid) { | ||
1835 | |||
1836 | if (device_table[i].device_type == NVL_DEVICE(ioctrl)) { | ||
1837 | |||
1838 | g->nvlink.ioctrl_type = | ||
1839 | device_table[i].device_type; | ||
1840 | g->nvlink.ioctrl_base = | ||
1841 | device_table[i].pri_base_addr; | ||
1842 | continue; | ||
1843 | } | ||
1844 | |||
1845 | if (device_table[i].device_type == NVL_DEVICE(dlpl)) { | ||
1846 | |||
1847 | g->nvlink.dlpl_type = | ||
1848 | device_table[i].device_type; | ||
1849 | g->nvlink.dlpl_base[device_table[i].device_id] = | ||
1850 | device_table[i].pri_base_addr; | ||
1851 | g->nvlink.links[device_table[i].device_id].valid = true; | ||
1852 | g->nvlink.links[device_table[i].device_id].g = g; | ||
1853 | g->nvlink.links[device_table[i].device_id].dlpl_version = | ||
1854 | device_table[i].device_version; | ||
1855 | g->nvlink.links[device_table[i].device_id].dlpl_base = | ||
1856 | device_table[i].pri_base_addr; | ||
1857 | g->nvlink.links[device_table[i].device_id].intr_enum = | ||
1858 | device_table[i].intr_enum; | ||
1859 | g->nvlink.links[device_table[i].device_id].reset_enum = | ||
1860 | device_table[i].reset_enum; | ||
1861 | g->nvlink.links[device_table[i].device_id].link_id = | ||
1862 | device_table[i].device_id; | ||
1863 | |||
1864 | /* initiate the PLL master and slave link id to max */ | ||
1865 | g->nvlink.links[device_table[i].device_id].pll_master_link_id = | ||
1866 | NVLINK_MAX_LINKS_SW; | ||
1867 | g->nvlink.links[device_table[i].device_id].pll_slave_link_id = | ||
1868 | NVLINK_MAX_LINKS_SW; | ||
1869 | |||
1870 | /* Update Pll master */ | ||
1871 | if (device_table[i].pll_master) | ||
1872 | g->nvlink.links[device_table[i].device_id].pll_master_link_id = | ||
1873 | g->nvlink.links[device_table[i].device_id].link_id; | ||
1874 | else { | ||
1875 | g->nvlink.links[device_table[i].device_id].pll_master_link_id = | ||
1876 | device_table[i].pll_master_id; | ||
1877 | g->nvlink.links[device_table[i].device_id].pll_slave_link_id = | ||
1878 | g->nvlink.links[device_table[i].device_id].link_id; | ||
1879 | g->nvlink.links[device_table[i].pll_master_id].pll_slave_link_id = | ||
1880 | g->nvlink.links[device_table[i].device_id].link_id; | ||
1881 | } | ||
1882 | |||
1883 | available_links |= BIT(device_table[i].device_id); | ||
1884 | continue; | ||
1885 | } | ||
1886 | |||
1887 | if (device_table[i].device_type == NVL_DEVICE(nvltlc)) { | ||
1888 | |||
1889 | g->nvlink.tl_type = device_table[i].device_type; | ||
1890 | g->nvlink.tl_base[device_table[i].device_id] = | ||
1891 | device_table[i].pri_base_addr; | ||
1892 | g->nvlink.links[device_table[i].device_id].tl_base = | ||
1893 | device_table[i].pri_base_addr; | ||
1894 | g->nvlink.links[device_table[i].device_id].tl_version = | ||
1895 | device_table[i].device_version; | ||
1896 | continue; | ||
1897 | } | ||
1898 | |||
1899 | if (device_table[i].device_type == NVL_DEVICE(nvltlc)) { | ||
1900 | |||
1901 | g->nvlink.tl_type = device_table[i].device_type; | ||
1902 | g->nvlink.tl_base[device_table[i].device_id] = | ||
1903 | device_table[i].pri_base_addr; | ||
1904 | g->nvlink.links[device_table[i].device_id].tl_base = | ||
1905 | device_table[i].pri_base_addr; | ||
1906 | g->nvlink.links[device_table[i].device_id].tl_version = | ||
1907 | device_table[i].device_version; | ||
1908 | continue; | ||
1909 | } | ||
1910 | |||
1911 | if (device_table[i].device_type == NVL_DEVICE(ioctrlmif)) { | ||
1912 | |||
1913 | g->nvlink.mif_type = device_table[i].device_type; | ||
1914 | g->nvlink.mif_base[device_table[i].device_id] = | ||
1915 | device_table[i].pri_base_addr; | ||
1916 | g->nvlink.links[device_table[i].device_id].mif_base = | ||
1917 | device_table[i].pri_base_addr; | ||
1918 | g->nvlink.links[device_table[i].device_id].mif_version = | ||
1919 | device_table[i].device_version; | ||
1920 | continue; | ||
1921 | } | ||
1922 | |||
1923 | if (device_table[i].device_type == NVL_DEVICE(nvlipt)) { | ||
1924 | |||
1925 | g->nvlink.ipt_type = | ||
1926 | device_table[i].device_type; | ||
1927 | g->nvlink.ipt_base = | ||
1928 | device_table[i].pri_base_addr; | ||
1929 | g->nvlink.ipt_version = | ||
1930 | device_table[i].device_version; | ||
1931 | continue; | ||
1932 | } | ||
1933 | |||
1934 | if (device_table[i].device_type == NVL_DEVICE(minion)) { | ||
1935 | |||
1936 | g->nvlink.minion_type = | ||
1937 | device_table[i].device_type; | ||
1938 | g->nvlink.minion_base = | ||
1939 | device_table[i].pri_base_addr; | ||
1940 | g->nvlink.minion_version = | ||
1941 | device_table[i].device_version; | ||
1942 | continue; | ||
1943 | } | ||
1944 | |||
1945 | if (device_table[i].device_type == NVL_DEVICE(dlpl_multicast)) { | ||
1946 | |||
1947 | g->nvlink.dlpl_multicast_type = | ||
1948 | device_table[i].device_type; | ||
1949 | g->nvlink.dlpl_multicast_base = | ||
1950 | device_table[i].pri_base_addr; | ||
1951 | g->nvlink.dlpl_multicast_version = | ||
1952 | device_table[i].device_version; | ||
1953 | continue; | ||
1954 | } | ||
1955 | if (device_table[i].device_type == NVL_DEVICE(nvltlc_multicast)) { | ||
1956 | |||
1957 | g->nvlink.tl_multicast_type = | ||
1958 | device_table[i].device_type; | ||
1959 | g->nvlink.tl_multicast_base = | ||
1960 | device_table[i].pri_base_addr; | ||
1961 | g->nvlink.tl_multicast_version = | ||
1962 | device_table[i].device_version; | ||
1963 | continue; | ||
1964 | } | ||
1965 | |||
1966 | if (device_table[i].device_type == NVL_DEVICE(ioctrlmif_multicast)) { | ||
1967 | |||
1968 | g->nvlink.mif_multicast_type = | ||
1969 | device_table[i].device_type; | ||
1970 | g->nvlink.mif_multicast_base = | ||
1971 | device_table[i].pri_base_addr; | ||
1972 | g->nvlink.mif_multicast_version = | ||
1973 | device_table[i].device_version; | ||
1974 | continue; | ||
1975 | } | ||
1976 | |||
1977 | } | ||
1978 | } | ||
1979 | |||
1980 | g->nvlink.discovered_links = (u32) available_links; | ||
1981 | |||
1982 | nvgpu_log(g, gpu_dbg_nvlink, "Nvlink Tree:"); | ||
1983 | nvgpu_log(g, gpu_dbg_nvlink, "+ Available Links: 0x%08lx", available_links); | ||
1984 | nvgpu_log(g, gpu_dbg_nvlink, "+ Per-Link Devices:"); | ||
1985 | |||
1986 | for_each_set_bit(i, &available_links, 32) { | ||
1987 | nvgpu_log(g, gpu_dbg_nvlink, "-- Link %d Dl/Pl Base: 0x%08x TLC Base: 0x%08x MIF Base: 0x%08x", | ||
1988 | i, g->nvlink.dlpl_base[i], g->nvlink.tl_base[i], g->nvlink.mif_base[i]); | ||
1989 | } | ||
1990 | |||
1991 | nvgpu_log(g, gpu_dbg_nvlink, "+ IOCTRL Base: 0x%08x", g->nvlink.ioctrl_base); | ||
1992 | nvgpu_log(g, gpu_dbg_nvlink, "+ NVLIPT Base: 0x%08x", g->nvlink.ipt_base); | ||
1993 | nvgpu_log(g, gpu_dbg_nvlink, "+ MINION Base: 0x%08x", g->nvlink.minion_base); | ||
1994 | nvgpu_log(g, gpu_dbg_nvlink, "+ DLPL MCAST Base: 0x%08x", g->nvlink.dlpl_multicast_base); | ||
1995 | nvgpu_log(g, gpu_dbg_nvlink, "+ TLC MCAST Base: 0x%08x", g->nvlink.tl_multicast_base); | ||
1996 | nvgpu_log(g, gpu_dbg_nvlink, "+ MIF MCAST Base: 0x%08x", g->nvlink.mif_multicast_base); | ||
1997 | |||
1998 | if (!g->nvlink.minion_version) { | ||
1999 | nvgpu_err(g, "Unsupported MINION version"); | ||
2000 | |||
2001 | nvgpu_kfree(g, device_table); | ||
2002 | g->nvlink.device_table = NULL; | ||
2003 | g->nvlink.num_devices = 0; | ||
2004 | return -EINVAL; | ||
2005 | } | ||
2006 | |||
2007 | return err; | ||
2008 | } | ||
2009 | |||
2010 | /* | ||
2011 | * Query IOCTRL for device discovery | ||
2012 | */ | ||
2013 | int gv100_nvlink_discover_ioctrl(struct gk20a *g) | ||
2014 | { | ||
2015 | u32 i; | ||
2016 | struct nvgpu_nvlink_ioctrl_list *ioctrl_table; | ||
2017 | u32 table_entry; | ||
2018 | u32 devinfo_type; | ||
2019 | bool is_ioctrl = false; | ||
2020 | bool is_chain = false; | ||
2021 | u32 io_num_entries = 0; | ||
2022 | |||
2023 | ioctrl_table = nvgpu_kzalloc(g, top_device_info__size_1_v() * | ||
2024 | sizeof(struct nvgpu_nvlink_ioctrl_list)); | ||
2025 | |||
2026 | if (!ioctrl_table) { | ||
2027 | nvgpu_err(g, "failed to allocate memory for nvlink io table"); | ||
2028 | return -ENOMEM; | ||
2029 | } | ||
2030 | |||
2031 | for (i = 0; i < top_device_info__size_1_v(); i++) { | ||
2032 | table_entry = gk20a_readl(g, top_device_info_r(i)); | ||
2033 | |||
2034 | devinfo_type = top_device_info_entry_v(table_entry); | ||
2035 | |||
2036 | if (devinfo_type == top_device_info_entry_not_valid_v()) | ||
2037 | continue; | ||
2038 | |||
2039 | if (devinfo_type == top_device_info_entry_engine_type_v()) { | ||
2040 | if (top_device_info_type_enum_v(table_entry) == | ||
2041 | top_device_info_type_enum_ioctrl_v()) | ||
2042 | is_ioctrl = true; | ||
2043 | else { | ||
2044 | is_ioctrl = false; | ||
2045 | continue; | ||
2046 | } | ||
2047 | |||
2048 | if (top_device_info_chain_v(table_entry) != | ||
2049 | top_device_info_chain_enable_v()) | ||
2050 | break; | ||
2051 | |||
2052 | is_chain = true; | ||
2053 | ioctrl_table[io_num_entries].valid = true; | ||
2054 | continue; | ||
2055 | } | ||
2056 | |||
2057 | if (devinfo_type == top_device_info_entry_data_v()) { | ||
2058 | if (is_ioctrl && is_chain) { | ||
2059 | |||
2060 | ioctrl_table[io_num_entries].pri_base_addr = | ||
2061 | top_device_info_data_pri_base_v(table_entry) << | ||
2062 | top_device_info_data_pri_base_align_v(); | ||
2063 | |||
2064 | if (top_device_info_chain_v(table_entry) != | ||
2065 | top_device_info_chain_enable_v()) { | ||
2066 | is_chain = false; | ||
2067 | io_num_entries++; | ||
2068 | } | ||
2069 | } | ||
2070 | continue; | ||
2071 | } | ||
2072 | |||
2073 | if (devinfo_type == top_device_info_entry_enum_v()) { | ||
2074 | if (is_ioctrl && is_chain) { | ||
2075 | |||
2076 | ioctrl_table[io_num_entries].intr_enum = | ||
2077 | top_device_info_intr_v(table_entry); | ||
2078 | ioctrl_table[io_num_entries].reset_enum = | ||
2079 | top_device_info_reset_v(table_entry); | ||
2080 | |||
2081 | if (top_device_info_chain_v(table_entry) != | ||
2082 | top_device_info_chain_enable_v()) { | ||
2083 | is_chain = false; | ||
2084 | io_num_entries++; | ||
2085 | } | ||
2086 | } | ||
2087 | continue; | ||
2088 | } | ||
2089 | } | ||
2090 | |||
2091 | if (io_num_entries == 0 || !ioctrl_table[0].pri_base_addr) { | ||
2092 | nvgpu_err(g, "No NVLINK io found"); | ||
2093 | nvgpu_kfree(g, ioctrl_table); | ||
2094 | return -EINVAL; | ||
2095 | } | ||
2096 | |||
2097 | g->nvlink.ioctrl_table = ioctrl_table; | ||
2098 | g->nvlink.io_num_entries = io_num_entries; | ||
2099 | |||
2100 | |||
2101 | for (i =0; i < io_num_entries; i++) | ||
2102 | nvgpu_log(g, gpu_dbg_nvlink, | ||
2103 | "Device %d : Pri Base Addr = 0x%0x Intr = %d Reset = %d", | ||
2104 | i, ioctrl_table[i].pri_base_addr, ioctrl_table[i].intr_enum, | ||
2105 | ioctrl_table[i].reset_enum); | ||
2106 | |||
2107 | return 0; | ||
2108 | } | ||
2109 | |||
2110 | /* | ||
2111 | ******************************************************************************* | ||
2112 | * NVLINK API FUNCTIONS * | ||
2113 | ******************************************************************************* | ||
2114 | */ | ||
2115 | |||
2116 | /* | ||
2117 | * Performs link level initialization like phy inits, AN0 and interrupts | ||
2118 | */ | ||
2119 | |||
2120 | int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask) | ||
2121 | { | ||
2122 | int err; | ||
2123 | |||
2124 | err = gv100_nvlink_enable_links_pre_top(g, mask); | ||
2125 | if (err) { | ||
2126 | nvgpu_err(g, "Pre topology failed for links %lx", mask); | ||
2127 | return err; | ||
2128 | } | ||
2129 | |||
2130 | nvgpu_log(g, gpu_dbg_nvlink, "pretopology enabled: 0x%lx", | ||
2131 | mask & g->nvlink.enabled_links); | ||
2132 | err = gv100_nvlink_enable_links_post_top(g, mask); | ||
2133 | |||
2134 | return err; | ||
2135 | } | ||
2136 | |||
2137 | /* | ||
2138 | * Performs memory interface initialization | ||
2139 | */ | ||
2140 | |||
2141 | int gv100_nvlink_interface_init(struct gk20a *g) | ||
2142 | { | ||
2143 | unsigned long mask = g->nvlink.enabled_links; | ||
2144 | u32 link_id; | ||
2145 | |||
2146 | for_each_set_bit(link_id, &mask, 32) { | ||
2147 | gv100_nvlink_initialize_mif(g, link_id); | ||
2148 | gv100_nvlink_mif_intr_enable(g, link_id, true); | ||
2149 | } | ||
2150 | |||
2151 | return 0; | ||
2152 | } | ||
2153 | |||
2154 | int gv100_nvlink_reg_init(struct gk20a *g) | ||
2155 | { | ||
2156 | u32 i = 0; | ||
2157 | u32 count = 0; | ||
2158 | struct __nvlink_reginit *reg; | ||
2159 | enum nvgpu_nvlink_endp endp; | ||
2160 | int err; | ||
2161 | u32 link_id; | ||
2162 | unsigned long mask = g->nvlink.enabled_links; | ||
2163 | struct nvgpu_nvlink_link *link; | ||
2164 | |||
2165 | /* Apply automated reg init flow for PROD settings */ | ||
2166 | for_each_set_bit(link_id, &mask, 32) { | ||
2167 | |||
2168 | link = &g->nvlink.links[link_id]; | ||
2169 | if (!link->remote_info.is_connected) | ||
2170 | continue; | ||
2171 | |||
2172 | endp = link->remote_info.device_type; | ||
2173 | err = gv100_nvlink_get_tlc_reginit(endp, ®, &count); | ||
2174 | if (err) { | ||
2175 | nvgpu_err(g, "no reginit for endp=%u", endp); | ||
2176 | continue; | ||
2177 | } | ||
2178 | |||
2179 | for (i = 0; i < count; i++) { | ||
2180 | TLC_REG_WR32(g, link_id, reg->addr, reg->value); | ||
2181 | reg++; | ||
2182 | } | ||
2183 | } | ||
2184 | return 0; | ||
2185 | } | ||
2186 | |||
2187 | /* | ||
2188 | * Shutdown device. This should tear down Nvlink connection. | ||
2189 | * For now return. | ||
2190 | */ | ||
2191 | int gv100_nvlink_shutdown(struct gk20a *g) | ||
2192 | { | ||
2193 | return 0; | ||
2194 | } | ||
2195 | |||
2196 | /* | ||
2197 | * Get link state | ||
2198 | */ | ||
2199 | u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id) | ||
2200 | { | ||
2201 | return DLPL_REG_RD32(g, link_id, nvl_link_state_r()) & | ||
2202 | nvl_link_state_state_m(); | ||
2203 | } | ||
2204 | |||
2205 | /* Get link mode */ | ||
2206 | u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id) | ||
2207 | { | ||
2208 | u32 state; | ||
2209 | if (!(BIT(link_id) & g->nvlink.discovered_links)) | ||
2210 | return nvgpu_nvlink_link__last; | ||
2211 | |||
2212 | state = nvl_link_state_state_v( | ||
2213 | g->ops.nvlink.link_get_state(g, link_id)); | ||
2214 | |||
2215 | if (state == nvl_link_state_state_init_v()) | ||
2216 | return nvgpu_nvlink_link_off; | ||
2217 | if (state == nvl_link_state_state_hwcfg_v()) | ||
2218 | return nvgpu_nvlink_link_detect; | ||
2219 | if (state == nvl_link_state_state_swcfg_v()) | ||
2220 | return nvgpu_nvlink_link_safe; | ||
2221 | if (state == nvl_link_state_state_active_v()) | ||
2222 | return nvgpu_nvlink_link_hs; | ||
2223 | if (state == nvl_link_state_state_fault_v()) | ||
2224 | return nvgpu_nvlink_link_fault; | ||
2225 | if ((state == nvl_link_state_state_rcvy_ac_v()) || | ||
2226 | (state == nvl_link_state_state_rcvy_sw_v()) || | ||
2227 | (state == nvl_link_state_state_rcvy_rx_v())) | ||
2228 | return nvgpu_nvlink_link_recovery; | ||
2229 | |||
2230 | return nvgpu_nvlink_link_off; | ||
2231 | } | ||
2232 | |||
2233 | /* Set Link mode */ | ||
2234 | int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode) | ||
2235 | { | ||
2236 | u32 state; | ||
2237 | u32 reg; | ||
2238 | u32 err = 0; | ||
2239 | |||
2240 | nvgpu_log(g, gpu_dbg_nvlink, "link :%d, mode:%u", link_id, mode); | ||
2241 | |||
2242 | if (!(BIT(link_id) & g->nvlink.enabled_links)) | ||
2243 | return -EINVAL; | ||
2244 | |||
2245 | state = nvl_link_state_state_v( | ||
2246 | g->ops.nvlink.link_get_state(g, link_id)); | ||
2247 | |||
2248 | switch (mode) { | ||
2249 | case nvgpu_nvlink_link_safe: | ||
2250 | if (state == nvl_link_state_state_swcfg_v()) { | ||
2251 | nvgpu_warn(g, "link is already in safe mode"); | ||
2252 | break; | ||
2253 | } | ||
2254 | if (state == nvl_link_state_state_hwcfg_v()) { | ||
2255 | nvgpu_warn(g, "link is transitioning to safe mode"); | ||
2256 | break; | ||
2257 | } | ||
2258 | |||
2259 | if (state == nvl_link_state_state_init_v()) { | ||
2260 | /* Off to Safe transition */ | ||
2261 | reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); | ||
2262 | reg = set_field(reg, nvl_link_change_newstate_m(), | ||
2263 | nvl_link_change_newstate_hwcfg_f()); | ||
2264 | reg = set_field(reg, nvl_link_change_oldstate_mask_m(), | ||
2265 | nvl_link_change_oldstate_mask_dontcare_f()); | ||
2266 | reg = set_field(reg, nvl_link_change_action_m(), | ||
2267 | nvl_link_change_action_ltssm_change_f()); | ||
2268 | DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); | ||
2269 | } else if (state == nvl_link_state_state_active_v()) { | ||
2270 | /* TODO: | ||
2271 | * Disable PM first since we are moving out active | ||
2272 | * state | ||
2273 | */ | ||
2274 | reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); | ||
2275 | reg = set_field(reg, nvl_link_change_newstate_m(), | ||
2276 | nvl_link_change_newstate_swcfg_f()); | ||
2277 | reg = set_field(reg, nvl_link_change_oldstate_mask_m(), | ||
2278 | nvl_link_change_oldstate_mask_dontcare_f()); | ||
2279 | reg = set_field(reg, nvl_link_change_action_m(), | ||
2280 | nvl_link_change_action_ltssm_change_f()); | ||
2281 | DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); | ||
2282 | } | ||
2283 | break; | ||
2284 | |||
2285 | case nvgpu_nvlink_link_hs: | ||
2286 | if (state == nvl_link_state_state_active_v()) { | ||
2287 | nvgpu_err(g, "link is already in active mode"); | ||
2288 | break; | ||
2289 | } | ||
2290 | if (state == nvl_link_state_state_init_v()) { | ||
2291 | nvgpu_err(g, "link cannot be taken from init state"); | ||
2292 | return -EPERM; | ||
2293 | } | ||
2294 | |||
2295 | reg = DLPL_REG_RD32(g, link_id, nvl_link_change_r()); | ||
2296 | reg = set_field(reg, nvl_link_change_newstate_m(), | ||
2297 | nvl_link_change_newstate_active_f()); | ||
2298 | reg = set_field(reg, nvl_link_change_oldstate_mask_m(), | ||
2299 | nvl_link_change_oldstate_mask_dontcare_f()); | ||
2300 | reg = set_field(reg, nvl_link_change_action_m(), | ||
2301 | nvl_link_change_action_ltssm_change_f()); | ||
2302 | DLPL_REG_WR32(g, link_id, nvl_link_change_r(), reg); | ||
2303 | break; | ||
2304 | |||
2305 | case nvgpu_nvlink_link_off: | ||
2306 | if (state == nvl_link_state_state_active_v()) { | ||
2307 | nvgpu_err(g, "link cannot be taken from active to init"); | ||
2308 | return -EPERM; | ||
2309 | } | ||
2310 | if (state == nvl_link_state_state_init_v()) { | ||
2311 | nvgpu_err(g, "link already in init state"); | ||
2312 | } | ||
2313 | |||
2314 | /* GV100 UPHY is handled by MINION */ | ||
2315 | break; | ||
2316 | /* 1/8 th mode not supported */ | ||
2317 | case nvgpu_nvlink_link_enable_pm: | ||
2318 | case nvgpu_nvlink_link_disable_pm: | ||
2319 | return -EPERM; | ||
2320 | case nvgpu_nvlink_link_disable_err_detect: | ||
2321 | /* Disable Link interrupts */ | ||
2322 | gv100_nvlink_dlpl_intr_enable(g, link_id, false); | ||
2323 | break; | ||
2324 | case nvgpu_nvlink_link_lane_disable: | ||
2325 | err = gv100_nvlink_minion_lane_disable(g, link_id, true); | ||
2326 | break; | ||
2327 | case nvgpu_nvlink_link_lane_shutdown: | ||
2328 | err = gv100_nvlink_minion_lane_shutdown(g, link_id, true); | ||
2329 | break; | ||
2330 | default: | ||
2331 | nvgpu_err(g, "Unhandled mode %x", mode); | ||
2332 | break; | ||
2333 | } | ||
2334 | |||
2335 | return err; | ||
2336 | } | ||
2337 | |||
2338 | static u32 gv100_nvlink_link_sublink_check_change(struct gk20a *g, u32 link_id) | ||
2339 | { | ||
2340 | struct nvgpu_timeout timeout; | ||
2341 | u32 reg; | ||
2342 | |||
2343 | nvgpu_timeout_init(g, &timeout, | ||
2344 | NVLINK_SUBLINK_TIMEOUT_MS, NVGPU_TIMER_CPU_TIMER); | ||
2345 | /* Poll for sublink status */ | ||
2346 | do { | ||
2347 | reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); | ||
2348 | |||
2349 | if (nvl_sublink_change_status_v(reg) == | ||
2350 | nvl_sublink_change_status_done_v()) | ||
2351 | break; | ||
2352 | if (nvl_sublink_change_status_v(reg) == | ||
2353 | nvl_sublink_change_status_fault_v()) { | ||
2354 | nvgpu_err(g, "Fault detected in sublink change"); | ||
2355 | return -EFAULT; | ||
2356 | } | ||
2357 | nvgpu_udelay(5); | ||
2358 | } while(!nvgpu_timeout_expired_msg(&timeout, "timeout on sublink rdy")); | ||
2359 | |||
2360 | if (nvgpu_timeout_peek_expired(&timeout)) | ||
2361 | return -ETIMEDOUT; | ||
2362 | return-0; | ||
2363 | } | ||
2364 | |||
2365 | int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id, | ||
2366 | bool is_rx_sublink, u32 mode) | ||
2367 | { | ||
2368 | int err = 0; | ||
2369 | u32 rx_sublink_state, tx_sublink_state; | ||
2370 | u32 reg; | ||
2371 | |||
2372 | if (!(BIT(link_id) & g->nvlink.enabled_links)) | ||
2373 | return -EINVAL; | ||
2374 | |||
2375 | err = gv100_nvlink_link_sublink_check_change(g, link_id); | ||
2376 | if (err) | ||
2377 | return err; | ||
2378 | |||
2379 | rx_sublink_state = g->ops.nvlink.get_rx_sublink_state(g, link_id); | ||
2380 | tx_sublink_state = g->ops.nvlink.get_tx_sublink_state(g, link_id); | ||
2381 | |||
2382 | switch (mode) { | ||
2383 | case nvgpu_nvlink_sublink_tx_hs: | ||
2384 | if (tx_sublink_state == | ||
2385 | nvl_sl0_slsm_status_tx_primary_state_hs_v()) { | ||
2386 | nvgpu_err(g, " TX already in HS"); | ||
2387 | break; | ||
2388 | } else if (tx_sublink_state == | ||
2389 | nvl_sl0_slsm_status_tx_primary_state_off_v()) { | ||
2390 | nvgpu_err(g, "TX cannot be do from OFF to HS"); | ||
2391 | return -EPERM; | ||
2392 | } | ||
2393 | |||
2394 | reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); | ||
2395 | reg = set_field(reg, nvl_sublink_change_newstate_m(), | ||
2396 | nvl_sublink_change_newstate_hs_f()); | ||
2397 | reg = set_field(reg, nvl_sublink_change_sublink_m(), | ||
2398 | nvl_sublink_change_sublink_tx_f()); | ||
2399 | reg = set_field(reg, nvl_sublink_change_action_m(), | ||
2400 | nvl_sublink_change_action_slsm_change_f()); | ||
2401 | DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); | ||
2402 | |||
2403 | err = gv100_nvlink_link_sublink_check_change(g, link_id); | ||
2404 | if (err) { | ||
2405 | nvgpu_err(g, "Error in TX to HS"); | ||
2406 | return err; | ||
2407 | } | ||
2408 | break; | ||
2409 | case nvgpu_nvlink_sublink_tx_common: | ||
2410 | err = gv100_nvlink_minion_init_uphy(g, BIT(link_id), true); | ||
2411 | break; | ||
2412 | case nvgpu_nvlink_sublink_tx_common_disable: | ||
2413 | /* NOP */ | ||
2414 | break; | ||
2415 | case nvgpu_nvlink_sublink_tx_data_ready: | ||
2416 | err = gv100_nvlink_minion_data_ready_en(g, BIT(link_id), true); | ||
2417 | break; | ||
2418 | case nvgpu_nvlink_sublink_tx_prbs_en: | ||
2419 | err = gv100_nvlink_prbs_gen_en(g, BIT(link_id)); | ||
2420 | break; | ||
2421 | case nvgpu_nvlink_sublink_tx_safe: | ||
2422 | if (tx_sublink_state == | ||
2423 | nvl_sl0_slsm_status_tx_primary_state_safe_v()) { | ||
2424 | nvgpu_err(g, "TX already SAFE: %d", link_id); | ||
2425 | break; | ||
2426 | } | ||
2427 | |||
2428 | reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); | ||
2429 | reg = set_field(reg, nvl_sublink_change_newstate_m(), | ||
2430 | nvl_sublink_change_newstate_safe_f()); | ||
2431 | reg = set_field(reg, nvl_sublink_change_sublink_m(), | ||
2432 | nvl_sublink_change_sublink_tx_f()); | ||
2433 | reg = set_field(reg, nvl_sublink_change_action_m(), | ||
2434 | nvl_sublink_change_action_slsm_change_f()); | ||
2435 | DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); | ||
2436 | |||
2437 | err = gv100_nvlink_link_sublink_check_change(g, link_id); | ||
2438 | if (err) { | ||
2439 | nvgpu_err(g, "Error in TX to SAFE"); | ||
2440 | return err; | ||
2441 | } | ||
2442 | break; | ||
2443 | case nvgpu_nvlink_sublink_tx_off: | ||
2444 | if (tx_sublink_state == | ||
2445 | nvl_sl0_slsm_status_tx_primary_state_off_v()) { | ||
2446 | nvgpu_err(g, "TX already OFF: %d", link_id); | ||
2447 | break; | ||
2448 | } else if (tx_sublink_state == | ||
2449 | nvl_sl0_slsm_status_tx_primary_state_hs_v()) { | ||
2450 | nvgpu_err(g, " TX cannot go off from HS %d", link_id); | ||
2451 | return -EPERM; | ||
2452 | } | ||
2453 | |||
2454 | reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); | ||
2455 | reg = set_field(reg, nvl_sublink_change_newstate_m(), | ||
2456 | nvl_sublink_change_newstate_off_f()); | ||
2457 | reg = set_field(reg, nvl_sublink_change_sublink_m(), | ||
2458 | nvl_sublink_change_sublink_tx_f()); | ||
2459 | reg = set_field(reg, nvl_sublink_change_action_m(), | ||
2460 | nvl_sublink_change_action_slsm_change_f()); | ||
2461 | DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); | ||
2462 | |||
2463 | err = gv100_nvlink_link_sublink_check_change(g, link_id); | ||
2464 | if (err) { | ||
2465 | nvgpu_err(g, "Error in TX to OFF"); | ||
2466 | return err; | ||
2467 | } | ||
2468 | break; | ||
2469 | |||
2470 | /* RX modes */ | ||
2471 | case nvgpu_nvlink_sublink_rx_hs: | ||
2472 | case nvgpu_nvlink_sublink_rx_safe: | ||
2473 | break; | ||
2474 | case nvgpu_nvlink_sublink_rx_off: | ||
2475 | if (rx_sublink_state == | ||
2476 | nvl_sl1_slsm_status_rx_primary_state_off_v()) { | ||
2477 | nvgpu_err(g, "RX already OFF: %d", link_id); | ||
2478 | break; | ||
2479 | } else if (rx_sublink_state == | ||
2480 | nvl_sl1_slsm_status_rx_primary_state_hs_v()) { | ||
2481 | nvgpu_err(g, " RX cannot go off from HS %d", link_id); | ||
2482 | return -EPERM; | ||
2483 | } | ||
2484 | |||
2485 | reg = DLPL_REG_RD32(g, link_id, nvl_sublink_change_r()); | ||
2486 | reg = set_field(reg, nvl_sublink_change_newstate_m(), | ||
2487 | nvl_sublink_change_newstate_off_f()); | ||
2488 | reg = set_field(reg, nvl_sublink_change_sublink_m(), | ||
2489 | nvl_sublink_change_sublink_rx_f()); | ||
2490 | reg = set_field(reg, nvl_sublink_change_action_m(), | ||
2491 | nvl_sublink_change_action_slsm_change_f()); | ||
2492 | DLPL_REG_WR32(g, link_id, nvl_sublink_change_r(), reg); | ||
2493 | |||
2494 | err = gv100_nvlink_link_sublink_check_change(g, link_id); | ||
2495 | if (err) { | ||
2496 | nvgpu_err(g, "Error in RX to OFF"); | ||
2497 | return err; | ||
2498 | } | ||
2499 | break; | ||
2500 | case nvgpu_nvlink_sublink_rx_rxcal: | ||
2501 | err = gv100_nvlink_rxcal_en(g, BIT(link_id)); | ||
2502 | break; | ||
2503 | |||
2504 | default: | ||
2505 | if ((is_rx_sublink) && ((mode < nvgpu_nvlink_sublink_rx_hs) || | ||
2506 | (mode >= nvgpu_nvlink_sublink_rx__last))) { | ||
2507 | nvgpu_err(g, "Unsupported RX mode %u", mode); | ||
2508 | return -EINVAL; | ||
2509 | } | ||
2510 | if (mode >= nvgpu_nvlink_sublink_tx__last) { | ||
2511 | nvgpu_err(g, "Unsupported TX mode %u", mode); | ||
2512 | return -EINVAL; | ||
2513 | } | ||
2514 | nvgpu_err(g, "MODE %u", mode); | ||
2515 | } | ||
2516 | |||
2517 | if (err) | ||
2518 | nvgpu_err(g, " failed on set_sublink_mode"); | ||
2519 | return err; | ||
2520 | } | ||
2521 | |||
2522 | u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id, | ||
2523 | bool is_rx_sublink) | ||
2524 | { | ||
2525 | u32 state; | ||
2526 | |||
2527 | if (!(BIT(link_id) & g->nvlink.discovered_links)) { | ||
2528 | if (!is_rx_sublink) | ||
2529 | return nvgpu_nvlink_sublink_tx__last; | ||
2530 | return nvgpu_nvlink_sublink_rx__last; | ||
2531 | } | ||
2532 | |||
2533 | if (!is_rx_sublink) { | ||
2534 | state = g->ops.nvlink.get_tx_sublink_state(g, link_id); | ||
2535 | if (state == nvl_sl0_slsm_status_tx_primary_state_hs_v()) | ||
2536 | return nvgpu_nvlink_sublink_tx_hs; | ||
2537 | if (state == nvl_sl0_slsm_status_tx_primary_state_eighth_v()) | ||
2538 | return nvgpu_nvlink_sublink_tx_single_lane; | ||
2539 | if (state == nvl_sl0_slsm_status_tx_primary_state_safe_v()) | ||
2540 | return nvgpu_nvlink_sublink_tx_safe; | ||
2541 | if (state == nvl_sl0_slsm_status_tx_primary_state_off_v()) | ||
2542 | return nvgpu_nvlink_sublink_tx_off; | ||
2543 | return nvgpu_nvlink_sublink_tx_off; | ||
2544 | } else { | ||
2545 | state = g->ops.nvlink.get_rx_sublink_state(g, link_id); | ||
2546 | if (state == nvl_sl1_slsm_status_rx_primary_state_hs_v()) | ||
2547 | return nvgpu_nvlink_sublink_rx_hs; | ||
2548 | if (state == nvl_sl1_slsm_status_rx_primary_state_eighth_v()) | ||
2549 | return nvgpu_nvlink_sublink_rx_single_lane; | ||
2550 | if (state == nvl_sl1_slsm_status_rx_primary_state_safe_v()) | ||
2551 | return nvgpu_nvlink_sublink_rx_safe; | ||
2552 | if (state == nvl_sl1_slsm_status_rx_primary_state_off_v()) | ||
2553 | return nvgpu_nvlink_sublink_rx_off; | ||
2554 | return nvgpu_nvlink_sublink_rx_off; | ||
2555 | } | ||
2556 | return nvgpu_nvlink_sublink_tx__last; | ||
2557 | } | ||
2558 | |||
2559 | /* | ||
2560 | * Get TX sublink state | ||
2561 | */ | ||
2562 | u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id) | ||
2563 | { | ||
2564 | u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl0_slsm_status_tx_r()); | ||
2565 | |||
2566 | return nvl_sl0_slsm_status_tx_primary_state_v(reg); | ||
2567 | } | ||
2568 | |||
2569 | /* | ||
2570 | * Get RX sublink state | ||
2571 | */ | ||
2572 | u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id) | ||
2573 | { | ||
2574 | u32 reg = DLPL_REG_RD32(g, link_id, nvl_sl1_slsm_status_rx_r()); | ||
2575 | |||
2576 | return nvl_sl1_slsm_status_rx_primary_state_v(reg); | ||
2577 | } | ||
2578 | |||
2579 | /* | ||
2580 | * Performs nvlink device level initialization by discovering the topology | ||
2581 | * taking device out of reset, boot minion, set clocks up and common interrupts | ||
2582 | */ | ||
2583 | int gv100_nvlink_early_init(struct gk20a *g) | ||
2584 | { | ||
2585 | int err = 0; | ||
2586 | |||
2587 | if (!nvgpu_is_enabled(g, NVGPU_SUPPORT_NVLINK)) | ||
2588 | return -EINVAL; | ||
2589 | |||
2590 | err = nvgpu_bios_get_nvlink_config_data(g); | ||
2591 | if (err) { | ||
2592 | nvgpu_err(g, "failed to read nvlink vbios data"); | ||
2593 | goto nvlink_init_exit; | ||
2594 | } | ||
2595 | |||
2596 | err = g->ops.nvlink.discover_ioctrl(g); | ||
2597 | if (err) | ||
2598 | goto nvlink_init_exit; | ||
2599 | |||
2600 | /* Enable NVLINK in MC */ | ||
2601 | g->ops.mc.reset(g, mc_enable_nvlink_enabled_f()); | ||
2602 | |||
2603 | err = g->ops.nvlink.discover_link(g); | ||
2604 | if (err || g->nvlink.discovered_links == 0) { | ||
2605 | nvgpu_err(g, "No links available"); | ||
2606 | goto nvlink_init_exit; | ||
2607 | } | ||
2608 | |||
2609 | nvgpu_flcn_sw_init(g, FALCON_ID_MINION); | ||
2610 | |||
2611 | g->nvlink.discovered_links &= ~g->nvlink.link_disable_mask; | ||
2612 | nvgpu_log(g, gpu_dbg_nvlink, "link_disable_mask = 0x%08x (from VBIOS)", | ||
2613 | g->nvlink.link_disable_mask); | ||
2614 | |||
2615 | /* Links in reset should be removed from initialized link sw state */ | ||
2616 | g->nvlink.initialized_links &= __gv100_nvlink_get_link_reset_mask(g); | ||
2617 | |||
2618 | nvgpu_log(g, gpu_dbg_nvlink, "connected_links = 0x%08x (from DT)", | ||
2619 | g->nvlink.connected_links); | ||
2620 | |||
2621 | /* Track unconnected links */ | ||
2622 | g->nvlink.discovered_links &= g->nvlink.connected_links; | ||
2623 | |||
2624 | nvgpu_log(g, gpu_dbg_nvlink, "discovered_links = 0x%08x (combination)", | ||
2625 | g->nvlink.discovered_links); | ||
2626 | |||
2627 | if (hweight32(g->nvlink.discovered_links) > 1) { | ||
2628 | nvgpu_err(g, "more than one link enabled"); | ||
2629 | err = -EINVAL; | ||
2630 | goto nvlink_init_exit; | ||
2631 | } | ||
2632 | |||
2633 | /* For now set default speed */ | ||
2634 | g->nvlink.speed = nvgpu_nvlink_speed_default; | ||
2635 | |||
2636 | err = __gv100_nvlink_state_load_hal(g); | ||
2637 | if (err) { | ||
2638 | nvgpu_err(g, " failed Nvlink state load"); | ||
2639 | goto nvlink_init_exit; | ||
2640 | } | ||
2641 | err = gv100_nvlink_minion_configure_ac_coupling(g, | ||
2642 | g->nvlink.ac_coupling_mask, true); | ||
2643 | if (err) { | ||
2644 | nvgpu_err(g, " failed Nvlink state load"); | ||
2645 | goto nvlink_init_exit; | ||
2646 | } | ||
2647 | |||
2648 | /* Program clocks */ | ||
2649 | gv100_nvlink_prog_alt_clk(g); | ||
2650 | |||
2651 | nvlink_init_exit: | ||
2652 | return err; | ||
2653 | } | ||
2654 | |||
diff --git a/drivers/gpu/nvgpu/gv100/nvlink_gv100.h b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h new file mode 100644 index 00000000..4ac8b907 --- /dev/null +++ b/drivers/gpu/nvgpu/gv100/nvlink_gv100.h | |||
@@ -0,0 +1,60 @@ | |||
1 | /* | ||
2 | * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. | ||
3 | * | ||
4 | * Permission is hereby granted, free of charge, to any person obtaining a | ||
5 | * copy of this software and associated documentation files (the "Software"), | ||
6 | * to deal in the Software without restriction, including without limitation | ||
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | ||
8 | * and/or sell copies of the Software, and to permit persons to whom the | ||
9 | * Software is furnished to do so, subject to the following conditions: | ||
10 | * | ||
11 | * The above copyright notice and this permission notice shall be included in | ||
12 | * all copies or substantial portions of the Software. | ||
13 | * | ||
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | ||
17 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
18 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | ||
19 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER | ||
20 | * DEALINGS IN THE SOFTWARE. | ||
21 | */ | ||
22 | |||
23 | #ifndef NVGPU_NVLINK_GV100_H | ||
24 | #define NVGPU_NVLINK_GV100_H | ||
25 | |||
26 | struct gk20a; | ||
27 | |||
28 | #define MINION_REG_RD32(g, off) gk20a_readl(g, g->nvlink.minion_base + (off)) | ||
29 | #define MINION_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.minion_base + (off), (v)) | ||
30 | #define IOCTRL_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ioctrl_base + (off)) | ||
31 | #define IOCTRL_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ioctrl_base + (off), (v)); | ||
32 | #define MIF_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].mif_base + (off)) | ||
33 | #define MIF_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].mif_base + (off), (v)) | ||
34 | #define IPT_REG_RD32(g, off) gk20a_readl(g, g->nvlink.ipt_base + (off)) | ||
35 | #define IPT_REG_WR32(g, off, v) gk20a_writel(g, g->nvlink.ipt_base + (off), (v)) | ||
36 | #define TLC_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].tl_base + (off)) | ||
37 | #define TLC_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].tl_base + (off), (v)) | ||
38 | #define DLPL_REG_RD32(g, id, off) gk20a_readl(g, g->nvlink.links[(id)].dlpl_base + (off)) | ||
39 | #define DLPL_REG_WR32(g, id, off, v) gk20a_writel(g, g->nvlink.links[(id)].dlpl_base + (off), (v)) | ||
40 | |||
41 | int gv100_nvlink_discover_ioctrl(struct gk20a *g); | ||
42 | int gv100_nvlink_discover_link(struct gk20a *g); | ||
43 | int gv100_nvlink_init(struct gk20a *g); | ||
44 | int gv100_nvlink_isr(struct gk20a *g); | ||
45 | /* API */ | ||
46 | int gv100_nvlink_link_early_init(struct gk20a *g, unsigned long mask); | ||
47 | u32 gv100_nvlink_link_get_mode(struct gk20a *g, u32 link_id); | ||
48 | u32 gv100_nvlink_link_get_state(struct gk20a *g, u32 link_id); | ||
49 | int gv100_nvlink_link_set_mode(struct gk20a *g, u32 link_id, u32 mode); | ||
50 | u32 gv100_nvlink_link_get_sublink_mode(struct gk20a *g, u32 link_id, | ||
51 | bool is_rx_sublink); | ||
52 | u32 gv100_nvlink_link_get_tx_sublink_state(struct gk20a *g, u32 link_id); | ||
53 | u32 gv100_nvlink_link_get_rx_sublink_state(struct gk20a *g, u32 link_id); | ||
54 | int gv100_nvlink_link_set_sublink_mode(struct gk20a *g, u32 link_id, | ||
55 | bool is_rx_sublink, u32 mode); | ||
56 | int gv100_nvlink_interface_init(struct gk20a *g); | ||
57 | int gv100_nvlink_reg_init(struct gk20a *g); | ||
58 | int gv100_nvlink_shutdown(struct gk20a *g); | ||
59 | int gv100_nvlink_early_init(struct gk20a *g); | ||
60 | #endif | ||
diff --git a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c index 89f3f0b9..97ab7aab 100644 --- a/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/fifo_gv11b.c | |||
@@ -482,7 +482,7 @@ static int gv11b_fifo_poll_eng_ctx_status(struct gk20a *g, u32 id, | |||
482 | eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); | 482 | eng_stat = gk20a_readl(g, fifo_engine_status_r(act_eng_id)); |
483 | ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); | 483 | ctx_stat = fifo_engine_status_ctx_status_v(eng_stat); |
484 | 484 | ||
485 | if (gv11b_mc_is_stall_and_eng_intr_pending(g, act_eng_id)) { | 485 | if (g->ops.mc.is_stall_and_eng_intr_pending(g, act_eng_id)) { |
486 | stall_intr = true; | 486 | stall_intr = true; |
487 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, | 487 | nvgpu_log(g, gpu_dbg_info | gpu_dbg_intr, |
488 | "stall intr set, " | 488 | "stall intr set, " |
diff --git a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c index 6fdac85a..5282af05 100644 --- a/drivers/gpu/nvgpu/gv11b/hal_gv11b.c +++ b/drivers/gpu/nvgpu/gv11b/hal_gv11b.c | |||
@@ -658,6 +658,8 @@ static const struct gpu_ops gv11b_ops = { | |||
658 | .boot_0 = gk20a_mc_boot_0, | 658 | .boot_0 = gk20a_mc_boot_0, |
659 | .is_intr1_pending = mc_gp10b_is_intr1_pending, | 659 | .is_intr1_pending = mc_gp10b_is_intr1_pending, |
660 | .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, | 660 | .is_intr_hub_pending = gv11b_mc_is_intr_hub_pending, |
661 | .is_stall_and_eng_intr_pending = | ||
662 | gv11b_mc_is_stall_and_eng_intr_pending, | ||
661 | }, | 663 | }, |
662 | .debug = { | 664 | .debug = { |
663 | .show_dump = gk20a_debug_show_dump, | 665 | .show_dump = gk20a_debug_show_dump, |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h index c59a6e6b..a57ea97a 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h +++ b/drivers/gpu/nvgpu/include/nvgpu/hw/gv100/hw_minion_gv100.h | |||
@@ -892,6 +892,14 @@ static inline u32 minion_nvlink_link_intr_code_dlreq_f(void) | |||
892 | { | 892 | { |
893 | return 0x2U; | 893 | return 0x2U; |
894 | } | 894 | } |
895 | static inline u32 minion_nvlink_link_intr_code_pmdisabled_v(void) | ||
896 | { | ||
897 | return 0x00000003U; | ||
898 | } | ||
899 | static inline u32 minion_nvlink_link_intr_code_pmdisabled_f(void) | ||
900 | { | ||
901 | return 0x3U; | ||
902 | } | ||
895 | static inline u32 minion_nvlink_link_intr_subcode_f(u32 v) | 903 | static inline u32 minion_nvlink_link_intr_subcode_f(u32 v) |
896 | { | 904 | { |
897 | return (v & 0xffU) << 8U; | 905 | return (v & 0xffU) << 8U; |
diff --git a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h index 48851ff1..18e0aab6 100644 --- a/drivers/gpu/nvgpu/include/nvgpu/nvlink.h +++ b/drivers/gpu/nvgpu/include/nvgpu/nvlink.h | |||
@@ -182,11 +182,10 @@ struct nvgpu_nvlink_dev { | |||
182 | u8 train_at_boot; | 182 | u8 train_at_boot; |
183 | u32 ac_coupling_mask; | 183 | u32 ac_coupling_mask; |
184 | 184 | ||
185 | u32 init_disabled_links; | ||
186 | u32 connected_links; | 185 | u32 connected_links; |
187 | u32 initialized_links; | 186 | u32 initialized_links; |
188 | u32 enabled_links; | 187 | u32 enabled_links; |
189 | u32 topology_connected_links; | 188 | u32 init_pll_done; |
190 | 189 | ||
191 | enum nvgpu_nvlink_speed speed; | 190 | enum nvgpu_nvlink_speed speed; |
192 | 191 | ||
@@ -195,12 +194,18 @@ struct nvgpu_nvlink_dev { | |||
195 | u32 hshub_config1; | 194 | u32 hshub_config1; |
196 | u32 hshub_config2; | 195 | u32 hshub_config2; |
197 | u32 hshub_config6; | 196 | u32 hshub_config6; |
197 | |||
198 | /* tlc cached errors */ | ||
199 | u32 tlc_rx_err_status_0[NVLINK_MAX_LINKS_SW]; | ||
200 | u32 tlc_rx_err_status_1[NVLINK_MAX_LINKS_SW]; | ||
201 | u32 tlc_tx_err_status_0[NVLINK_MAX_LINKS_SW]; | ||
202 | |||
198 | /* priv struct */ | 203 | /* priv struct */ |
199 | void *priv; | 204 | void *priv; |
200 | }; | 205 | }; |
201 | 206 | ||
202 | 207 | ||
203 | u32 nvgpu_nvlink_enumerate(struct gk20a *g); | 208 | int nvgpu_nvlink_enumerate(struct gk20a *g); |
204 | u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); | 209 | int nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off); |
205 | u32 nvgpu_nvlink_probe(struct gk20a *g); | 210 | int nvgpu_nvlink_probe(struct gk20a *g); |
206 | #endif | 211 | #endif |