summaryrefslogtreecommitdiffstats
path: root/drivers/gpu/nvgpu/common/linux/nvlink.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/nvgpu/common/linux/nvlink.c')
-rw-r--r--drivers/gpu/nvgpu/common/linux/nvlink.c531
1 files changed, 531 insertions, 0 deletions
diff --git a/drivers/gpu/nvgpu/common/linux/nvlink.c b/drivers/gpu/nvgpu/common/linux/nvlink.c
new file mode 100644
index 00000000..eea6eda7
--- /dev/null
+++ b/drivers/gpu/nvgpu/common/linux/nvlink.c
@@ -0,0 +1,531 @@
1/*
2 * Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see <http://www.gnu.org/licenses/>.
15 */
16
17#include <gk20a/gk20a.h>
18#include <nvgpu/nvlink.h>
19#include <nvgpu/enabled.h>
20#include "module.h"
21
22#ifdef CONFIG_TEGRA_NVLINK
23#include <linux/platform/tegra/tegra-nvlink.h>
24#endif
25
26#ifdef CONFIG_TEGRA_NVLINK
27
28/*
29 * WAR: use this function to find detault link, as only one is supported
30 * on the library for now
31 * Returns NVLINK_MAX_LINKS_SW on failure
32 */
33static u32 __nvgpu_nvlink_get_link(struct nvlink_device *ndev)
34{
35 u32 link_id;
36 struct gk20a *g = (struct gk20a *) ndev->priv;
37
38 if (!g)
39 return NVLINK_MAX_LINKS_SW;
40
41 /* Lets find the detected link */
42 if (g->nvlink.initialized_links)
43 link_id = fls(g->nvlink.initialized_links);
44 else
45 return NVLINK_MAX_LINKS_SW;
46
47 if (g->nvlink.links[link_id].remote_info.is_connected)
48 return link_id;
49
50 return NVLINK_MAX_LINKS_SW;
51}
52static int nvgpu_nvlink_early_init(struct nvlink_device *ndev)
53{
54 struct gk20a *g = (struct gk20a *) ndev->priv;
55 int err;
56
57 /* For now master topology is the only one supported */
58 if (!ndev->is_master) {
59 nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink,
60 "dGPU is not master of Nvlink link");
61 return -EINVAL;
62 }
63
64 err = g->ops.nvlink.early_init(g);
65 return err;
66}
67
68static int nvgpu_nvlink_link_early_init(struct nvlink_device *ndev)
69{
70 struct gk20a *g = (struct gk20a *) ndev->priv;
71 int err;
72 u32 link_id;
73
74 /*
75 * First check the topology and setup connectivity
76 * HACK: we are only enabling one link for now!!!
77 */
78 link_id = fls(g->nvlink.discovered_links);
79 g->nvlink.links[link_id].remote_info.is_connected = true;
80
81 err = g->ops.nvlink.link_early_init(g, BIT(link_id));
82
83 if (err == 0) {
84 g->nvlink.links[link_id].priv = (void *) &(ndev->link);
85 ndev->link.priv = (void *) g;
86 }
87 return err;
88}
89
90static int nvgpu_nvlink_interface_init(struct nvlink_device *ndev)
91{
92 int err;
93 struct gk20a *g = (struct gk20a *) ndev->priv;
94
95 err = g->ops.nvlink.interface_init(g);
96 return err;
97}
98
99static int nvgpu_nvlink_shutdown(struct nvlink_device *ndev)
100{
101 int err;
102 struct gk20a *g = (struct gk20a *) ndev->priv;
103
104 err = g->ops.nvlink.shutdown(g);
105 return 0;
106}
107
108static int nvgpu_nvlink_reg_init(struct nvlink_device *ndev)
109{
110 struct gk20a *g = (struct gk20a *) ndev->priv;
111 int err;
112
113 err = g->ops.nvlink.reg_init(g);
114
115 return err;
116}
117
118static u32 nvgpu_nvlink_get_link_mode(struct nvlink_device *ndev)
119{
120 struct gk20a *g = (struct gk20a *) ndev->priv;
121 u32 link_id;
122 u32 mode;
123
124 link_id = __nvgpu_nvlink_get_link(ndev);
125 if (link_id == NVLINK_MAX_LINKS_SW)
126 return -EINVAL;
127
128 mode = g->ops.nvlink.link_get_mode(g, link_id);
129
130 switch (mode) {
131 case nvgpu_nvlink_link_off:
132 return NVLINK_LINK_OFF;
133 case nvgpu_nvlink_link_hs:
134 return NVLINK_LINK_HS;
135 case nvgpu_nvlink_link_safe:
136 return NVLINK_LINK_SAFE;
137 case nvgpu_nvlink_link_fault:
138 return NVLINK_LINK_FAULT;
139 case nvgpu_nvlink_link_recovery:
140 return NVLINK_LINK_RECOVERY;
141 case nvgpu_nvlink_link_detect:
142 return NVLINK_LINK_DETECT;
143 case nvgpu_nvlink_link_reset:
144 return NVLINK_LINK_RESET;
145 case nvgpu_nvlink_link_enable_pm:
146 return NVLINK_LINK_ENABLE_PM;
147 case nvgpu_nvlink_link_disable_pm:
148 return NVLINK_LINK_DISABLE_PM;
149 case nvgpu_nvlink_link_disable_err_detect:
150 return NVLINK_LINK_DISABLE_ERR_DETECT;
151 case nvgpu_nvlink_link_lane_disable:
152 return NVLINK_LINK_LANE_DISABLE;
153 case nvgpu_nvlink_link_lane_shutdown:
154 return NVLINK_LINK_LANE_SHUTDOWN;
155 default:
156 nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink,
157 "unsupported mode %u", mode);
158 }
159
160 return NVLINK_LINK_OFF;
161}
162
163static u32 nvgpu_nvlink_get_link_state(struct nvlink_device *ndev)
164{
165 struct gk20a *g = (struct gk20a *) ndev->priv;
166 u32 link_id;
167
168 link_id = __nvgpu_nvlink_get_link(ndev);
169 if (link_id == NVLINK_MAX_LINKS_SW)
170 return -EINVAL;
171
172 return g->ops.nvlink.link_get_state(g, link_id);
173}
174
175static int nvgpu_nvlink_set_link_mode(struct nvlink_device *ndev, u32 mode)
176{
177
178 struct gk20a *g = (struct gk20a *) ndev->priv;
179 u32 link_id;
180 u32 mode_sw;
181
182 link_id = __nvgpu_nvlink_get_link(ndev);
183 if (link_id == NVLINK_MAX_LINKS_SW)
184 return -EINVAL;
185
186 switch (mode) {
187 case NVLINK_LINK_OFF:
188 mode_sw = nvgpu_nvlink_link_off;
189 break;
190 case NVLINK_LINK_HS:
191 mode_sw = nvgpu_nvlink_link_hs;
192 break;
193 case NVLINK_LINK_SAFE:
194 mode_sw = nvgpu_nvlink_link_safe;
195 break;
196 case NVLINK_LINK_FAULT:
197 mode_sw = nvgpu_nvlink_link_fault;
198 break;
199 case NVLINK_LINK_RECOVERY:
200 mode_sw = nvgpu_nvlink_link_recovery;
201 break;
202 case NVLINK_LINK_DETECT:
203 mode_sw = nvgpu_nvlink_link_detect;
204 break;
205 case NVLINK_LINK_RESET:
206 mode_sw = nvgpu_nvlink_link_reset;
207 break;
208 case NVLINK_LINK_ENABLE_PM:
209 mode_sw = nvgpu_nvlink_link_enable_pm;
210 break;
211 case NVLINK_LINK_DISABLE_PM:
212 mode_sw = nvgpu_nvlink_link_disable_pm;
213 break;
214 case NVLINK_LINK_LANE_DISABLE:
215 mode_sw = nvgpu_nvlink_link_lane_disable;
216 break;
217 case NVLINK_LINK_LANE_SHUTDOWN:
218 mode_sw = nvgpu_nvlink_link_lane_shutdown;
219 break;
220 default:
221 mode_sw = nvgpu_nvlink_link_off;
222 }
223
224 return g->ops.nvlink.link_set_mode(g, link_id, mode_sw);
225}
226
227static void nvgpu_nvlink_get_tx_sublink_state(struct nvlink_device *ndev,
228 u32 *state)
229{
230 struct gk20a *g = (struct gk20a *) ndev->priv;
231 u32 link_id;
232
233 link_id = __nvgpu_nvlink_get_link(ndev);
234 if (link_id == NVLINK_MAX_LINKS_SW)
235 return;
236 if (state)
237 *state = g->ops.nvlink.get_tx_sublink_state(g, link_id);
238}
239
240static void nvgpu_nvlink_get_rx_sublink_state(struct nvlink_device *ndev,
241 u32 *state)
242{
243 struct gk20a *g = (struct gk20a *) ndev->priv;
244 u32 link_id;
245
246 link_id = __nvgpu_nvlink_get_link(ndev);
247 if (link_id == NVLINK_MAX_LINKS_SW)
248 return;
249 if (state)
250 *state = g->ops.nvlink.get_rx_sublink_state(g, link_id);
251}
252
253static u32 nvgpu_nvlink_get_sublink_mode(struct nvlink_device *ndev,
254 bool is_rx_sublink)
255{
256 struct gk20a *g = (struct gk20a *) ndev->priv;
257 u32 link_id;
258 u32 mode;
259
260 link_id = __nvgpu_nvlink_get_link(ndev);
261 if (link_id == NVLINK_MAX_LINKS_SW)
262 return -EINVAL;
263
264 mode = g->ops.nvlink.get_sublink_mode(g, link_id, is_rx_sublink);
265 switch (mode) {
266 case nvgpu_nvlink_sublink_tx_hs:
267 return NVLINK_TX_HS;
268 case nvgpu_nvlink_sublink_tx_off:
269 return NVLINK_TX_OFF;
270 case nvgpu_nvlink_sublink_tx_single_lane:
271 return NVLINK_TX_SINGLE_LANE;
272 case nvgpu_nvlink_sublink_tx_safe:
273 return NVLINK_TX_SAFE;
274 case nvgpu_nvlink_sublink_tx_enable_pm:
275 return NVLINK_TX_ENABLE_PM;
276 case nvgpu_nvlink_sublink_tx_disable_pm:
277 return NVLINK_TX_DISABLE_PM;
278 case nvgpu_nvlink_sublink_tx_common:
279 return NVLINK_TX_COMMON;
280 case nvgpu_nvlink_sublink_tx_common_disable:
281 return NVLINK_TX_COMMON_DISABLE;
282 case nvgpu_nvlink_sublink_tx_data_ready:
283 return NVLINK_TX_DATA_READY;
284 case nvgpu_nvlink_sublink_tx_prbs_en:
285 return NVLINK_TX_PRBS_EN;
286 case nvgpu_nvlink_sublink_rx_hs:
287 return NVLINK_RX_HS;
288 case nvgpu_nvlink_sublink_rx_enable_pm:
289 return NVLINK_RX_ENABLE_PM;
290 case nvgpu_nvlink_sublink_rx_disable_pm:
291 return NVLINK_RX_DISABLE_PM;
292 case nvgpu_nvlink_sublink_rx_single_lane:
293 return NVLINK_RX_SINGLE_LANE;
294 case nvgpu_nvlink_sublink_rx_safe:
295 return NVLINK_RX_SAFE;
296 case nvgpu_nvlink_sublink_rx_off:
297 return NVLINK_RX_OFF;
298 case nvgpu_nvlink_sublink_rx_rxcal:
299 return NVLINK_RX_RXCAL;
300 default:
301 nvgpu_log(g, gpu_dbg_nvlink, "Unsupported mode: %u", mode);
302 break;
303 }
304
305 if (is_rx_sublink)
306 return NVLINK_RX_OFF;
307 return NVLINK_TX_OFF;
308}
309
310static int nvgpu_nvlink_set_sublink_mode(struct nvlink_device *ndev,
311 bool is_rx_sublink, u32 mode)
312{
313 struct gk20a *g = (struct gk20a *) ndev->priv;
314 u32 link_id;
315 u32 mode_sw;
316
317 link_id = __nvgpu_nvlink_get_link(ndev);
318 if (link_id == NVLINK_MAX_LINKS_SW)
319 return -EINVAL;
320
321 if (!is_rx_sublink) {
322 switch (mode) {
323 case NVLINK_TX_HS:
324 mode_sw = nvgpu_nvlink_sublink_tx_hs;
325 break;
326 case NVLINK_TX_ENABLE_PM:
327 mode_sw = nvgpu_nvlink_sublink_tx_enable_pm;
328 break;
329 case NVLINK_TX_DISABLE_PM:
330 mode_sw = nvgpu_nvlink_sublink_tx_disable_pm;
331 break;
332 case NVLINK_TX_SINGLE_LANE:
333 mode_sw = nvgpu_nvlink_sublink_tx_single_lane;
334 break;
335 case NVLINK_TX_SAFE:
336 mode_sw = nvgpu_nvlink_sublink_tx_safe;
337 break;
338 case NVLINK_TX_OFF:
339 mode_sw = nvgpu_nvlink_sublink_tx_off;
340 break;
341 case NVLINK_TX_COMMON:
342 mode_sw = nvgpu_nvlink_sublink_tx_common;
343 break;
344 case NVLINK_TX_COMMON_DISABLE:
345 mode_sw = nvgpu_nvlink_sublink_tx_common_disable;
346 break;
347 case NVLINK_TX_DATA_READY:
348 mode_sw = nvgpu_nvlink_sublink_tx_data_ready;
349 break;
350 case NVLINK_TX_PRBS_EN:
351 mode_sw = nvgpu_nvlink_sublink_tx_prbs_en;
352 break;
353 default:
354 return -EINVAL;
355 }
356 } else {
357 switch (mode) {
358 case NVLINK_RX_HS:
359 mode_sw = nvgpu_nvlink_sublink_rx_hs;
360 break;
361 case NVLINK_RX_ENABLE_PM:
362 mode_sw = nvgpu_nvlink_sublink_rx_enable_pm;
363 break;
364 case NVLINK_RX_DISABLE_PM:
365 mode_sw = nvgpu_nvlink_sublink_rx_disable_pm;
366 break;
367 case NVLINK_RX_SINGLE_LANE:
368 mode_sw = nvgpu_nvlink_sublink_rx_single_lane;
369 break;
370 case NVLINK_RX_SAFE:
371 mode_sw = nvgpu_nvlink_sublink_rx_safe;
372 break;
373 case NVLINK_RX_OFF:
374 mode_sw = nvgpu_nvlink_sublink_rx_off;
375 break;
376 case NVLINK_RX_RXCAL:
377 mode_sw = nvgpu_nvlink_sublink_rx_rxcal;
378 break;
379 default:
380 return -EINVAL;
381 }
382 }
383
384 return g->ops.nvlink.set_sublink_mode(g, link_id, is_rx_sublink,
385 mode_sw);
386}
387#endif
388
389u32 nvgpu_nvlink_enumerate(struct gk20a *g)
390{
391 u32 err = -ENODEV;
392#ifdef CONFIG_TEGRA_NVLINK
393 struct nvlink_device *ndev;
394
395 ndev = (struct nvlink_device *) g->nvlink.priv;
396
397 if (ndev)
398 err = nvlink_enumerate(ndev);
399#endif
400 return err;
401}
402
403u32 nvgpu_nvlink_train(struct gk20a *g, u32 link_id, bool from_off)
404{
405 u32 err = -ENODEV;
406#ifdef CONFIG_TEGRA_NVLINK
407 struct nvlink_device *ndev;
408
409 ndev = (struct nvlink_device *) g->nvlink.priv;
410
411 if (!ndev)
412 return -ENODEV;
413
414 /* Check if the link is connected */
415 if (!g->nvlink.links[link_id].remote_info.is_connected)
416 return -ENODEV;
417
418 if (from_off)
419 return nvlink_transition_intranode_conn_off_to_safe(ndev);
420 return nvlink_train_intranode_conn_safe_to_hs(ndev);
421
422#endif
423 return err;
424}
425
426u32 nvgpu_nvlink_probe(struct gk20a *g)
427{
428#ifdef CONFIG_TEGRA_NVLINK
429 u32 err = 0;
430 struct device_node *np = nvgpu_get_node(g);
431 struct device_node *nvlink_np = NULL, *endp_np = NULL;
432 struct nvlink_device *ndev;
433 u32 phys_link_id;
434
435 /* Parse DT */
436 if (np) {
437 nvlink_np = of_get_child_by_name(np, "nvidia,nvlink");
438 if (nvlink_np)
439 endp_np = of_get_child_by_name(np, "endpoint");
440 }
441
442 if (!endp_np) {
443 nvgpu_log(g, gpu_dbg_info | gpu_dbg_nvlink,
444 "No Nvlink DT detected");
445 return -ENODEV;
446 }
447
448 /* Allocating structures */
449 ndev = nvgpu_kzalloc(g, sizeof(struct nvlink_device));
450 if (!ndev) {
451 nvgpu_err(g, "OOM while allocating nvlink device struct");
452 return -ENOMEM;
453 }
454
455 ndev->priv = (void *) g;
456 g->nvlink.priv = (void *) ndev;
457
458 /* Parse DT structure to detect endpoint topology */
459 of_property_read_u32(endp_np, "local_dev_id", &ndev->device_id);
460 of_property_read_u32(endp_np, "local_link_id", &ndev->link.link_id);
461 ndev->is_master = of_property_read_bool(endp_np, "is_master");
462 of_property_read_u32(endp_np, "remote_dev_id",
463 &ndev->link.remote_dev_info.device_id);
464 of_property_read_u32(endp_np, "remote_link_id",
465 &ndev->link.remote_dev_info.link_id);
466 of_property_read_u32(endp_np, "physical_link",
467 &phys_link_id);
468
469 g->nvlink.topology_connected_links = BIT(phys_link_id);
470
471 mutex_init(&ndev->init_state_mutex);
472
473 /* Check that we are in dGPU mode */
474 if (ndev->device_id != NVLINK_ENDPT_GV100) {
475 nvgpu_err(g, "Local nvlink device is not dGPU");
476 err = -EINVAL;
477 goto free_nvlink;
478 }
479
480 err = nvlink_set_init_state(ndev, NVLINK_DEV_OFF);
481 if (err) {
482 nvgpu_err(g, "Error initalizing device state to OFF");
483 goto free_nvlink;
484 }
485
486 /* Fill in device struct */
487 ndev->dev_ops.dev_early_init = nvgpu_nvlink_early_init;
488 ndev->dev_ops.dev_interface_init = nvgpu_nvlink_interface_init;
489 ndev->dev_ops.dev_reg_init = nvgpu_nvlink_reg_init;
490 ndev->dev_ops.dev_shutdown = nvgpu_nvlink_shutdown;
491
492 /* Fill in the link struct */
493 ndev->link.device_id = ndev->device_id;
494 ndev->link.mode = NVLINK_LINK_OFF;
495 ndev->link.link_ops.get_link_mode = nvgpu_nvlink_get_link_mode;
496 ndev->link.link_ops.set_link_mode = nvgpu_nvlink_set_link_mode;
497 ndev->link.link_ops.get_sublink_mode = nvgpu_nvlink_get_sublink_mode;
498 ndev->link.link_ops.set_sublink_mode = nvgpu_nvlink_set_sublink_mode;
499 ndev->link.link_ops.get_link_state = nvgpu_nvlink_get_link_state;
500 ndev->link.link_ops.get_tx_sublink_state =
501 nvgpu_nvlink_get_tx_sublink_state;
502 ndev->link.link_ops.get_rx_sublink_state =
503 nvgpu_nvlink_get_rx_sublink_state;
504 ndev->link.link_ops.link_early_init =
505 nvgpu_nvlink_link_early_init;
506
507 /* Register device with core driver*/
508 err = nvlink_register_device(ndev);
509 if (err) {
510 nvgpu_err(g, "failed on nvlink device registration");
511 goto free_nvlink;
512 }
513
514 /* Register link with core driver */
515 err = nvlink_register_link(&ndev->link);
516 if (err) {
517 nvgpu_err(g, "failed on nvlink link registration");
518 goto free_nvlink;
519 }
520
521 /* Enable NVLINK support */
522 __nvgpu_set_enabled(g, NVGPU_SUPPORT_NVLINK, true);
523free_nvlink:
524 nvgpu_kfree(g, ndev);
525 return err;
526
527#else
528 return -ENODEV;
529#endif
530}
531
generated by cgit v1.2.2 (git 2.25.0) at 2024-07-16 15:32:11 -0400